Skip to content

Commit 7b154ff

Browse files
cigolplcigolpl
committed
set configuration mutex | create data dir only on write | indices list pagination
1 parent 44c1a46 commit 7b154ff

File tree

6 files changed

+84
-26
lines changed

6 files changed

+84
-26
lines changed

‎cpp/itemsjs.cpp

Lines changed: 55 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,16 +29,14 @@ namespace fs = ghc::filesystem;
2929
using namespace std;
3030

3131
const char *DELIMITERS = "!\"#$%&'()*+,-./:;<=>?@\[\\]^_`{|}~\n\v\f\r ";
32-
unsigned int WRITER_FLAGS = MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC | MDB_NOMETASYNC | MDB_NORDAHEAD;
32+
//unsigned int WRITER_FLAGS = MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC | MDB_NOMETASYNC | MDB_NORDAHEAD;
3333
//unsigned int WRITER_FLAGS = 0;
3434

3535
// mutex is used so MDB_NOTLS not needed
36-
unsigned int READER_FLAGS = MDB_NOLOCK | MDB_NOTLS;
36+
//unsigned int READER_FLAGS = MDB_NOLOCK | MDB_NOTLS;
3737
//unsigned int READER_FLAGS = 0;
3838
const auto DB_SIZE = 100UL * 1024UL * 1024UL * 1024UL;
3939

40-
std::mutex super_mutex;
41-
4240
int fast_atoi( const char * str ) {
4341
int val = 0;
4442
while( *str ) {
@@ -52,7 +50,7 @@ int fast_atoi( const char * str ) {
5250
*/
5351
std::tuple<std::string, std::optional<Roaring>, std::optional<Roaring>> itemsjs::search_facets(const char *&index_path, nlohmann::json input, nlohmann::json filters_array, nlohmann::json config, nlohmann::json facets_fields, std::optional<Roaring> query_ids, bool testing = false) {
5452

55-
// @TODO make unordered
53+
// @TODO make unordered (if faster)
5654
std::map<string, std::map<string, Roaring>> filters_indexes;
5755
std::map<string, std::map<string, Roaring>> not_filters_indexes;
5856
std::map<string, Roaring> combination;
@@ -642,9 +640,46 @@ std::vector<int> itemsjs::sort_index_2(const char *&index_path, const Roaring &i
642640
return sorted_ids;
643641
}
644642

643+
void itemsjs::set_configuration(const char *&index_path, const std::string& json) {
644+
645+
if (!fs::is_directory(index_path) || !fs::exists(index_path)) {
646+
fs::create_directory(index_path);
647+
}
648+
649+
string file_lock_path = (string)index_path + "/lock";
650+
std::ofstream output(file_lock_path.c_str());
651+
652+
try {
653+
auto lock = std::make_unique<boost::interprocess::file_lock>(file_lock_path.c_str());
654+
if (!lock->try_lock()) {
655+
lock->lock();
656+
}
657+
658+
auto env = lmdb::env::create();
659+
660+
env.set_mapsize(DB_SIZE);
661+
env.set_max_dbs(20);
662+
env.open(index_path, 0, 0664);
663+
664+
auto wtxn = lmdb::txn::begin(env);
665+
auto dbi = lmdb::dbi::open(wtxn, nullptr);
666+
dbi.put(wtxn, "configuration", json.c_str());
667+
668+
wtxn.commit();
669+
env.close();
670+
671+
} catch (const boost::interprocess::interprocess_exception &e) {
672+
cout << "There was an error with setting configuration interprocess mutex" << endl;
673+
cout << e.what() << endl;
674+
}
675+
}
645676

646677
std::string itemsjs::index(const char *&index_path, string json_path, const string& json_string, vector<string> &faceted_fields, std::vector<std::string> &sorting_fields, bool append = true) {
647678

679+
if (!fs::is_directory(index_path) || !fs::exists(index_path)) {
680+
fs::create_directory(index_path);
681+
}
682+
648683
string file_lock_path = (string)index_path + "/lock";
649684
std::ofstream output(file_lock_path.c_str());
650685

@@ -994,6 +1029,20 @@ void itemsjs::DeleteItemWrapped(const Napi::CallbackInfo& info) {
9941029
itemsjs::delete_item(index_path, id);
9951030
}
9961031

1032+
void itemsjs::SetConfigurationWrapped(const Napi::CallbackInfo& info) {
1033+
1034+
Napi::String index_name = info[0].As<Napi::String>();
1035+
string name_a(index_name.ToString());
1036+
const char *index_path = name_a.c_str();
1037+
1038+
// json already stringified
1039+
Napi::String json_object = info[1].As<Napi::String>();
1040+
string json_string(json_object.ToString());
1041+
1042+
itemsjs::set_configuration(index_path, json_string);
1043+
}
1044+
1045+
9971046
void itemsjs::LoadSortIndexWrapped(const Napi::CallbackInfo& info) {
9981047

9991048
Napi::String index_name = info[0].As<Napi::String>();
@@ -1237,6 +1286,7 @@ Napi::String itemsjs::IndexWrapped(const Napi::CallbackInfo& info) {
12371286
Napi::Object itemsjs::Init(Napi::Env env, Napi::Object exports) {
12381287

12391288
exports.Set("delete_item", Napi::Function::New(env, itemsjs::DeleteItemWrapped));
1289+
exports.Set("set_configuration", Napi::Function::New(env, itemsjs::SetConfigurationWrapped));
12401290
exports.Set("sort_index", Napi::Function::New(env, itemsjs::SortIndexWrapped));
12411291
exports.Set("sort_index_2", Napi::Function::New(env, itemsjs::SortIndex2Wrapped));
12421292
exports.Set("load_sort_index", Napi::Function::New(env, itemsjs::LoadSortIndexWrapped));

‎cpp/itemsjs.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ namespace itemsjs {
1414
std::vector<int> sort_index(const char *&index_path, const Roaring &ids, std::string field, std::string order, int offset, int limit);
1515
std::vector<int> sort_index_2(const char *&index_path, const Roaring &ids, std::string field, std::string order, int offset, int limit);
1616
void load_sort_index(const char *&index_path, std::vector<std::string> &sorting_fields);
17+
void set_configuration(const char *&index_path, const std::string& json);
18+
void SetConfigurationWrapped(const Napi::CallbackInfo& info);
1719
void delete_item(const char *&index_path, int id);
1820
void DeleteItemWrapped(const Napi::CallbackInfo& info);
1921
void LoadSortIndexWrapped(const Napi::CallbackInfo& info);

‎package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "itemsjs-server-optimized",
3-
"version": "2.1.4",
3+
"version": "2.1.6",
44
"description": "ItemsJS on steroid",
55
"main": "lib/index.js",
66
"scripts": {

‎src/facets.js

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ const addon = require('./addon');
1717
*/
1818
var Facets = function() {
1919
this.config = {};
20-
this.indexes_cache;
2120
};
2221

2322
Facets.prototype = {
@@ -123,28 +122,37 @@ Facets.prototype = {
123122
storage.setConfiguration(index_path, configuration);
124123
},
125124

126-
list_indexes: async function() {
125+
list_indexes: async function(params) {
126+
127+
params = params || {};
128+
var per_page = parseInt(params.per_page || 12);
129+
var page = parseInt(params.page || 1);
127130

128131
var output = [];
129132

130133
fs.readdirSync('./data').forEach(file => {
131134

132135
if (file.match(/\.mdb$/)) {
133-
output.push(file.slice(0, -4));
136+
output.push({
137+
index_name: file.slice(0, -4)
138+
})
134139
}
135140
});
136141

137142
return {
138-
data: output
143+
pagination: {
144+
per_page: per_page,
145+
page: page,
146+
total: output.length
147+
},
148+
data: output.slice((page - 1) * per_page, page * per_page)
139149
}
140150
},
141151

142-
143152
configuration: function(index_path) {
144153
return storage.getConfiguration(index_path);
145154
},
146155

147-
148156
/*
149157
* split query for normalized tokens
150158
*/

‎src/index.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@ module.exports = function itemsjs() {
2626

2727
/**
2828
*/
29-
list_indexes: async function() {
30-
return facets.list_indexes();
29+
list_indexes: async function(params) {
30+
return facets.list_indexes(params);
3131
},
3232

3333
load_sort_index: function(index_name) {

‎src/storage.js

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,6 @@ const MAX_DBS = 30;
1313

1414
var openDB = function(index_path) {
1515

16-
if (!fs.existsSync(index_path)) {
17-
fs.mkdirSync(index_path);
18-
}
19-
2016
const env = new lmdb.Env();
2117

2218
env.open({
@@ -141,16 +137,18 @@ module.exports.deleteConfiguration = function(index_path, configuration) {
141137

142138
module.exports.setConfiguration = function(index_path, configuration) {
143139

144-
var open = openDB(index_path);
145-
var dbi = open.dbi;
146-
var env = open.env;
140+
addon.set_configuration(index_path, JSON.stringify(configuration));
147141

148-
var txn = env.beginTxn();
149-
var binary = txn.putBinary(dbi, new Buffer.from('configuration'), new Buffer.from(JSON.stringify(configuration)));
150-
txn.commit();
142+
//var open = openDB(index_path);
143+
//var dbi = open.dbi;
144+
//var env = open.env;
151145

152-
dbi.close();
153-
env.close();
146+
//var txn = env.beginTxn();
147+
//var binary = txn.putBinary(dbi, new Buffer.from('configuration'), new Buffer.from(JSON.stringify(configuration)));
148+
//txn.commit();
149+
150+
//dbi.close();
151+
//env.close();
154152
}
155153

156154
module.exports.getConfiguration = function(index_path) {

0 commit comments

Comments
 (0)