diff --git a/binding.gyp b/binding.gyp index 1b73c74..326dffe 100644 --- a/binding.gyp +++ b/binding.gyp @@ -40,7 +40,6 @@ 'sources': [ "./src/cpp_util.cpp", "./src/node_util.cpp", - "./src/normalizationcache.cpp", "./src/memorycache.cpp", "./src/rocksdbcache.cpp", "./src/coalesce.cpp", diff --git a/src/binding.cpp b/src/binding.cpp index 712354e..c366f89 100644 --- a/src/binding.cpp +++ b/src/binding.cpp @@ -785,7 +785,6 @@ extern "C" { static void start(Handle target) { JSMemoryCache::Initialize(target); JSRocksDBCache::Initialize(target); - NormalizationCache::Initialize(target); Nan::SetMethod(target, "coalesce", JSCoalesce); } } diff --git a/src/binding.hpp b/src/binding.hpp index d6d6280..78adbb6 100644 --- a/src/binding.hpp +++ b/src/binding.hpp @@ -4,7 +4,6 @@ #include "coalesce.hpp" #include "memorycache.hpp" #include "node_util.hpp" -#include "normalizationcache.hpp" #include "rocksdbcache.hpp" #pragma clang diagnostic push @@ -61,9 +60,9 @@ using JSRocksDBCache = JSCache; using JSMemoryCache = JSCache; template -intarray __get(JSCache* c, std::string phrase, langfield_type langfield); +intarray __get(JSCache* c, const std::string& phrase, langfield_type langfield); template -intarray __getmatching(JSCache* c, std::string phrase, bool match_prefixes, langfield_type langfield); +intarray __getmatching(JSCache* c, const std::string& phrase, bool match_prefixes, langfield_type langfield); struct CoalesceBaton : carmen::noncopyable { uv_work_t request; diff --git a/src/cpp_util.hpp b/src/cpp_util.hpp index 30c8d45..8c179ce 100644 --- a/src/cpp_util.hpp +++ b/src/cpp_util.hpp @@ -288,7 +288,7 @@ inline void packVec(intarray const& varr, std::unique_ptr const& db db->Put(rocksdb::WriteOptions(), key, message); } -// rocksdb is also used in memorycache, and normalizationcache +// rocksdb is also used in memorycache rocksdb::Status OpenDB(const rocksdb::Options& options, const std::string& name, std::unique_ptr& dbptr); rocksdb::Status OpenForReadOnlyDB(const rocksdb::Options& options, const std::string& name, std::unique_ptr& dbptr); diff --git a/src/memorycache.cpp b/src/memorycache.cpp index dd508a5..bdcc0b7 100644 --- a/src/memorycache.cpp +++ b/src/memorycache.cpp @@ -4,12 +4,13 @@ namespace carmen { -intarray MemoryCache::__get(std::string phrase, langfield_type langfield) { +intarray MemoryCache::__get(const std::string& phrase, langfield_type langfield) { arraycache const& cache = this->cache_; intarray array; + std::string phrase_with_langfield = phrase; - add_langfield(phrase, langfield); - auto aitr = cache.find(phrase); + add_langfield(phrase_with_langfield, langfield); + auto aitr = cache.find(phrase_with_langfield); if (aitr != cache.end()) { array = aitr->second; } @@ -17,13 +18,13 @@ intarray MemoryCache::__get(std::string phrase, langfield_type langfield) { return array; } -intarray MemoryCache::__getmatching(std::string phrase, bool match_prefixes, langfield_type langfield) { +intarray MemoryCache::__getmatching(const std::string& phrase_ref, bool match_prefixes, langfield_type langfield) { intarray array; + std::string phrase = phrase_ref; if (!match_prefixes) phrase.push_back(LANGFIELD_SEPARATOR); size_t phrase_length = phrase.length(); const char* phrase_data = phrase.data(); - // Load values from memory cache for (auto const& item : this->cache_) { diff --git a/src/memorycache.hpp b/src/memorycache.hpp index ad72c35..0f80be9 100644 --- a/src/memorycache.hpp +++ b/src/memorycache.hpp @@ -15,11 +15,11 @@ class MemoryCache { void _set(std::string key_id, std::vector, langfield_type langfield, bool append); - std::vector _get(std::string phrase, std::vector languages); + std::vector _get(std::string& phrase, std::vector languages); std::vector _getmatching(std::string phrase, bool match_prefixes, std::vector languages); - std::vector __get(std::string phrase, langfield_type langfield); - std::vector __getmatching(std::string phrase, bool match_prefixes, langfield_type langfield); + std::vector __get(const std::string& phrase, langfield_type langfield); + std::vector __getmatching(const std::string& phrase_ref, bool match_prefixes, langfield_type langfield); arraycache cache_; }; diff --git a/src/normalizationcache.cpp b/src/normalizationcache.cpp deleted file mode 100644 index 9c2f46b..0000000 --- a/src/normalizationcache.cpp +++ /dev/null @@ -1,396 +0,0 @@ - -#include "normalizationcache.hpp" - -namespace carmen { - -using namespace v8; - -Nan::Persistent NormalizationCache::constructor; - -/** - * NormalizationCache represents an one-to-many integer-to-integer - * mapping where each integer is assumed to be the position of a given term - * in a lexicographically-sorted vocabulary. The purpose of the cache is to capture - * equivalencies between different elements in the dictionary, such that further metadata - * (e.g., in a RocksDBCache) can be stored only for the canonical form of a given name. - * The structure is stored using a RocksDB database on disk. - * @class NormalizationCache - * - */ - -void NormalizationCache::Initialize(Handle target) { - Nan::HandleScope scope; - Local t = Nan::New(NormalizationCache::New); - t->InstanceTemplate()->SetInternalFieldCount(1); - t->SetClassName(Nan::New("NormalizationCache").ToLocalChecked()); - Nan::SetPrototypeMethod(t, "get", get); - Nan::SetPrototypeMethod(t, "getPrefixRange", getprefixrange); - Nan::SetPrototypeMethod(t, "getAll", getall); - Nan::SetPrototypeMethod(t, "writeBatch", writebatch); - - target->Set(Nan::New("NormalizationCache").ToLocalChecked(), t->GetFunction()); - constructor.Reset(t); -} - -NormalizationCache::NormalizationCache() - : ObjectWrap(), - db() {} - -NormalizationCache::~NormalizationCache() {} - -class UInt32Comparator : public rocksdb::Comparator { - public: - UInt32Comparator(const UInt32Comparator&) = delete; - UInt32Comparator& operator=(const UInt32Comparator&) = delete; - UInt32Comparator() = default; - - int Compare(const rocksdb::Slice& a, const rocksdb::Slice& b) const override { - uint32_t ia = 0, ib = 0; - if (a.size() >= sizeof(uint32_t)) memcpy(&ia, a.data(), sizeof(uint32_t)); - if (b.size() >= sizeof(uint32_t)) memcpy(&ib, b.data(), sizeof(uint32_t)); - - if (ia < ib) return -1; - if (ia > ib) return +1; - return 0; - } - - const char* Name() const override { return "UInt32Comparator"; } - -// these function signatures are mandated by rocksdb, so suppress warnings about not -// using all the parameters -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wunused-parameter" - void FindShortestSeparator(std::string* start, const rocksdb::Slice& limit) const override {} - void FindShortSuccessor(std::string* key) const override {} -#pragma clang diagnostic pop -}; -UInt32Comparator UInt32ComparatorInstance; - -/** - * Constructor for NormalizationCache pointing to an on-disk RocksDB database - * to be used for reading or writing. - * - * @name NormalizationCache - * @memberof NormalizationCache - * @param {String} filename - * @param {String} read-only - * @returns {Object} - * @example - * const cache = require('@mapbox/carmen-cache'); - * const nc = new cache.NormalizationCache('file.norm.rocksdb', false); - * - */ - -NAN_METHOD(NormalizationCache::New) { - if (!info.IsConstructCall()) { - return Nan::ThrowTypeError("Cannot call constructor as function, you need to use 'new' keyword"); - } - try { - if (info.Length() < 2) { - return Nan::ThrowTypeError("expected arguments 'filename' and 'read-only'"); - } - if (!info[0]->IsString()) { - return Nan::ThrowTypeError("first argument 'filename' must be a String"); - } - if (!info[1]->IsBoolean()) { - return Nan::ThrowTypeError("second argument 'read-only' must be a Boolean"); - } - - Nan::Utf8String utf8_filename(info[0]); - if (utf8_filename.length() < 1) { - return Nan::ThrowTypeError("first arg must be a String"); - } - std::string filename(*utf8_filename); - bool read_only = info[1]->BooleanValue(); - - std::unique_ptr db; - rocksdb::Options options; - options.create_if_missing = true; - options.comparator = &UInt32ComparatorInstance; - - rocksdb::Status status; - if (read_only) { - status = OpenForReadOnlyDB(options, filename, db); - } else { - status = OpenDB(options, filename, db); - } - - if (!status.ok()) { - return Nan::ThrowTypeError("unable to open rocksdb file for normalization cache"); - } - NormalizationCache* im = new NormalizationCache(); - im->db = std::move(db); - im->Wrap(info.This()); - info.This()->Set(Nan::New("id").ToLocalChecked(), info[0]); - info.GetReturnValue().Set(info.This()); - return; - } catch (std::exception const& ex) { - return Nan::ThrowTypeError(ex.what()); - } -} - -/** - * retrieve the indices of the canonical labels for the index of a given non-canonical label - * - * @name get - * @memberof NormalizationCache - * @param {Number} id - * @returns {Array} - * @example - * const cache = require('@mapbox/carmen-cache'); - * const nc = new cache.NormalizationCache('file.norm.rocksdb', true); - * - * // for a normalization cache for the dictionary ['main st', 'main street'] - * // where 'main st' is canonical - * const canonical = nc.get(1); // returns [0] - */ - -NAN_METHOD(NormalizationCache::get) { - if (info.Length() < 1) { - return Nan::ThrowTypeError("expected one info: id"); - } - if (!info[0]->IsNumber()) { - return Nan::ThrowTypeError("first arg must be a Number"); - } - - uint32_t id = static_cast(info[0]->IntegerValue()); - std::string sid(reinterpret_cast(&id), sizeof(uint32_t)); - - NormalizationCache* c = node::ObjectWrap::Unwrap(info.This()); - std::shared_ptr db = c->db; - - std::string message; - bool found; - rocksdb::Status s = db->Get(rocksdb::ReadOptions(), sid, &message); - found = s.ok(); - - size_t message_length = message.size(); - if (found && message_length >= sizeof(uint32_t)) { - Local out = Nan::New(); - uint32_t entry; - for (uint32_t i = 0; i * sizeof(uint32_t) < message_length; i++) { - memcpy(&entry, message.data() + (i * sizeof(uint32_t)), sizeof(uint32_t)); - out->Set(i, Nan::New(entry)); - } - info.GetReturnValue().Set(out); - return; - } else { - info.GetReturnValue().Set(Nan::Undefined()); - return; - } -} - -/** - * given that in a lexicographically sorted list, all terms that share a prefix - * are grouped together, this function retrieves the indices of all canonical forms - * of all terms that share a given prefix as indicated by the index of the first term - * in the shared prefix list and the number of terms that share a prefix, for which the canonical - * form does not also share that same prefix - * - * @name getPrefixRange - * @memberof NormalizationCache - * @param {Number} start_id - * @param {Number} count - * @param {Number} [scan_max] - the maximum number of entries to scan - * @param {Number} [return_max] - the maximum number of indices to return - * @returns {Array} - * @example - * const cache = require('@mapbox/carmen-cache'); - * const nc = new cache.NormalizationCache('file.norm.rocksdb', true); - * - * // for a normalization cache for the dictionary - * // ['saint marks ave', 'saint peters ave', 'st marks ave', 'st peters ave'] - * // where the 'st ...' forms are canonical - * const canonical = nc.getPrefixRange(0, 2); // looks up all the canonical - * // forms for things that begin with - * // 'saint' - */ - -NAN_METHOD(NormalizationCache::getprefixrange) { - if (info.Length() < 1) { - return Nan::ThrowTypeError("expected at least two info: start_id, count, [scan_max], [return_max]"); - } - if (!info[0]->IsNumber()) { - return Nan::ThrowTypeError("first arg must be a Number"); - } - if (!info[1]->IsNumber()) { - return Nan::ThrowTypeError("second arg must be a Number"); - } - - uint32_t scan_max = 100; - uint32_t return_max = 10; - if (info.Length() > 2) { - if (!info[2]->IsNumber()) { - return Nan::ThrowTypeError("third arg, if supplied, must be a Number"); - } else { - scan_max = static_cast(info[2]->IntegerValue()); - } - } - if (info.Length() > 3) { - if (!info[3]->IsNumber()) { - return Nan::ThrowTypeError("third arg, if supplied, must be a Number"); - } else { - return_max = static_cast(info[3]->IntegerValue()); - } - } - - uint32_t start_id = static_cast(info[0]->IntegerValue()); - std::string sid(reinterpret_cast(&start_id), sizeof(uint32_t)); - uint32_t count = static_cast(info[1]->IntegerValue()); - uint32_t ceiling = start_id + count; - - uint32_t scan_count = 0, return_count = 0; - - Local out = Nan::New(); - unsigned out_idx = 0; - - NormalizationCache* c = node::ObjectWrap::Unwrap(info.This()); - std::shared_ptr db = c->db; - - std::unique_ptr rit(db->NewIterator(rocksdb::ReadOptions())); - for (rit->Seek(sid); rit->Valid(); rit->Next()) { - std::string skey = rit->key().ToString(); - uint32_t key; - memcpy(&key, skey.data(), sizeof(uint32_t)); - - if (key >= ceiling) break; - - uint32_t val; - std::string svalue = rit->value().ToString(); - for (uint32_t offset = 0; offset < svalue.length(); offset += sizeof(uint32_t)) { - memcpy(&val, svalue.data() + offset, sizeof(uint32_t)); - if (val < start_id || val >= ceiling) { - out->Set(out_idx++, Nan::New(val)); - - return_count++; - if (return_count >= return_max) break; - } - } - - scan_count++; - if (scan_count >= scan_max) break; - } - - info.GetReturnValue().Set(out); - return; -} - -/** - * retrieve the entire contents of a NormalizationCache, as an array of arrays - * - * @name getAll - * @memberof NormalizationCache - * @returns {Array} - * @example - * const cache = require('@mapbox/carmen-cache'); - * const nc = new cache.NormalizationCache('file.norm.rocksdb', true); - * - * // for a normalization cache for the dictionary - * // ['saint marks ave', 'saint peters ave', 'st marks ave', 'st peters ave'] - * // where the 'st ...' forms are canonical - * const canonical = nc.getAll() // returns [[0, [2]], [1, [3]]] - */ -NAN_METHOD(NormalizationCache::getall) { - Local out = Nan::New(); - unsigned out_idx = 0; - - NormalizationCache* c = node::ObjectWrap::Unwrap(info.This()); - std::shared_ptr db = c->db; - - std::unique_ptr rit(db->NewIterator(rocksdb::ReadOptions())); - for (rit->SeekToFirst(); rit->Valid(); rit->Next()) { - std::string skey = rit->key().ToString(); - uint32_t key = *reinterpret_cast(skey.data()); - - std::string svalue = rit->value().ToString(); - - Local row = Nan::New(); - row->Set(0, Nan::New(key)); - - Local vals = Nan::New(); - uint32_t entry; - for (uint32_t i = 0; i * sizeof(uint32_t) < svalue.length(); i++) { - memcpy(&entry, svalue.data() + (i * sizeof(uint32_t)), sizeof(uint32_t)); - vals->Set(i, Nan::New(entry)); - } - - row->Set(1, vals); - - out->Set(out_idx++, row); - } - - info.GetReturnValue().Set(out); - return; -} - -/** - * bulk-set the contents of a NormalizationCache to an array of arrays - * - * @name writeBatch - * @memberof NormalizationCache - * @param {Array} data - the values to be written to the cache, in the form [[from, [to, to, ...]], ...] - * @returns {Array} - * @example - * const cache = require('@mapbox/carmen-cache'); - * const nc = new cache.NormalizationCache('file.norm.rocksdb', true); - * - * // for a normalization cache for the dictionary - * // ['saint marks ave', 'saint peters ave', 'st marks ave', 'st peters ave'] - * // where the 'st ...' forms are canonical - * nc.writeBatch([[0, [2]], [1, [3]]]); - */ -NAN_METHOD(NormalizationCache::writebatch) { - if (info.Length() < 1) { - return Nan::ThrowTypeError("expected one info: data"); - } - if (!info[0]->IsArray()) { - return Nan::ThrowTypeError("first arg must be an Array"); - } - Local data = Local::Cast(info[0]); - if (data->IsNull() || data->IsUndefined()) { - return Nan::ThrowTypeError("an array expected for first argument"); - } - - NormalizationCache* c = node::ObjectWrap::Unwrap(info.This()); - std::shared_ptr db = c->db; - - rocksdb::WriteBatch batch; - for (uint32_t i = 0; i < data->Length(); i++) { - if (!data->Get(i)->IsArray()) return Nan::ThrowTypeError("second argument must be an array of arrays"); - Local row = Local::Cast(data->Get(i)); - - if (row->Length() != 2) return Nan::ThrowTypeError("each element must have two values"); - - uint32_t key = static_cast(row->Get(0)->IntegerValue()); - std::string skey(reinterpret_cast(&key), sizeof(uint32_t)); - - std::string svalue(""); - - Local nvalue = row->Get(1); - uint32_t ivalue; - if (nvalue->IsNumber()) { - ivalue = static_cast(nvalue->IntegerValue()); - svalue.append(reinterpret_cast(&ivalue), sizeof(uint32_t)); - } else if (nvalue->IsArray()) { - Local nvalue_arr = Local::Cast(nvalue); - if (!nvalue_arr->IsNull() && !nvalue_arr->IsUndefined()) { - for (uint32_t j = 0; j < nvalue_arr->Length(); j++) { - ivalue = static_cast(nvalue_arr->Get(j)->IntegerValue()); - svalue.append(reinterpret_cast(&ivalue), sizeof(uint32_t)); - } - } else { - return Nan::ThrowTypeError("values should be either numbers or arrays of numbers"); - } - } else { - return Nan::ThrowTypeError("values should be either numbers or arrays of numbers"); - } - - batch.Put(skey, svalue); - } - db->Write(rocksdb::WriteOptions(), &batch); - - info.GetReturnValue().Set(Nan::Undefined()); - return; -} - -} // namespace carmen diff --git a/src/normalizationcache.hpp b/src/normalizationcache.hpp deleted file mode 100644 index c02be88..0000000 --- a/src/normalizationcache.hpp +++ /dev/null @@ -1,45 +0,0 @@ -#ifndef __CARMEN_NORMALIZATIONCACHE_HPP__ -#define __CARMEN_NORMALIZATIONCACHE_HPP__ - -#include "cpp_util.hpp" - -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wunknown-pragmas" -#pragma clang diagnostic ignored "-Wconversion" -#pragma clang diagnostic ignored "-Wshadow" -#pragma clang diagnostic ignored "-Wsign-compare" -#pragma clang diagnostic ignored "-Wunused-local-typedef" -#pragma clang diagnostic ignored "-Wunused-parameter" -#pragma clang diagnostic ignored "-Wpadded" -#pragma clang diagnostic ignored "-Wold-style-cast" -#pragma clang diagnostic ignored "-Wsign-conversion" -#pragma clang diagnostic ignored "-Wshorten-64-to-32" - -#include "rocksdb/comparator.h" -#include "rocksdb/db.h" -#include "rocksdb/write_batch.h" -#include - -#pragma clang diagnostic pop - -namespace carmen { - -class NormalizationCache : public node::ObjectWrap { - public: - ~NormalizationCache(); - static Nan::Persistent constructor; - static void Initialize(v8::Handle target); - static NAN_METHOD(New); - static NAN_METHOD(get); - static NAN_METHOD(getprefixrange); - static NAN_METHOD(getall); - static NAN_METHOD(writebatch); - explicit NormalizationCache(); - void _ref() { Ref(); } - void _unref() { Unref(); } - std::shared_ptr db; -}; - -} // namespace carmen - -#endif // __CARMEN_NORMALIZATIONCACHE_HPP__ diff --git a/src/rocksdbcache.cpp b/src/rocksdbcache.cpp index a14f078..52a520c 100644 --- a/src/rocksdbcache.cpp +++ b/src/rocksdbcache.cpp @@ -4,12 +4,13 @@ namespace carmen { -intarray RocksDBCache::__get(std::string phrase, langfield_type langfield) { +intarray RocksDBCache::__get(const std::string& phrase, langfield_type langfield) { intarray array; + std::string phrase_with_langfield = phrase; - add_langfield(phrase, langfield); + add_langfield(phrase_with_langfield, langfield); std::string message; - rocksdb::Status s = db->Get(rocksdb::ReadOptions(), phrase, &message); + rocksdb::Status s = db->Get(rocksdb::ReadOptions(), phrase_with_langfield, &message); if (s.ok()) { decodeMessage(message, array); } @@ -17,10 +18,13 @@ intarray RocksDBCache::__get(std::string phrase, langfield_type langfield) { return array; } -intarray RocksDBCache::__getmatching(std::string phrase, bool match_prefixes, langfield_type langfield) { +intarray RocksDBCache::__getmatching(const std::string& phrase_ref, bool match_prefixes, langfield_type langfield) { intarray array; + std::string phrase = phrase_ref; - if (!match_prefixes) phrase.push_back(LANGFIELD_SEPARATOR); + if (!match_prefixes) { + phrase.push_back(LANGFIELD_SEPARATOR); + } size_t phrase_length = phrase.length(); // Load values from message cache diff --git a/src/rocksdbcache.hpp b/src/rocksdbcache.hpp index 4fb8e34..180bf8c 100644 --- a/src/rocksdbcache.hpp +++ b/src/rocksdbcache.hpp @@ -74,11 +74,9 @@ class RocksDBCache { bool pack(const std::string& filename); std::vector> list(); - std::vector _get(std::string phrase, std::vector languages); - std::vector _getmatching(std::string phrase, bool match_prefixes, std::vector languages); - std::vector __get(std::string phrase, langfield_type langfield); - std::vector __getmatching(std::string phrase, bool match_prefixes, langfield_type langfield); + std::vector __get(const std::string& phrase, langfield_type langfield); + std::vector __getmatching(const std::string& phrase_ref, bool match_prefixes, langfield_type langfield); std::shared_ptr db; }; diff --git a/test/normalize.test.js b/test/normalize.test.js deleted file mode 100644 index f4300fd..0000000 --- a/test/normalize.test.js +++ /dev/null @@ -1,123 +0,0 @@ -'use strict'; -const carmenCache = require('../index.js'); -const test = require('tape'); -const fs = require('fs'); - -const tmpdir = '/tmp/temp.' + Math.random().toString(36).substr(2, 5); -fs.mkdirSync(tmpdir); -let tmpidx = 0; -const tmpfile = function() { return tmpdir + '/' + (tmpidx++) + '.dat'; }; - -const words = [ - 'first street', - '1st st', - 'frank blvd', - 'frank boulevard', - 'fred road', - 'apple lane', - 'pear avenue', - 'pear ave', - 'burbarg', - 'buerbarg' -].sort(); - -const norm = { - 'first street': '1st st', - 'frank boulevard': 'frank blvd', - 'pear avenue': 'pear ave', - // for inconsistently normalized text, allow storing more than one possible normalization - 'burbarg': ['burbarg', 'buerbarg'] -}; - -const file = tmpfile(); - -// test creating the cache and writing it to disk -test('write/dump', (t) => { - const cache = new carmenCache.NormalizationCache(file, false); - - const map = []; - for (const key of Object.keys(norm).sort()) { - const val = (Array.isArray(norm[key]) ? norm[key] : [norm[key]]).map((x) => { return words.indexOf(x); }).sort(); - map.push([words.indexOf(key), val]); - } - - // These tests just illustrate what the mapping actually is storing. - // Note the mapping is based on index position of sorted text based - // on how dawg-cache stores. - t.deepEqual(words, [ - '1st st', - 'apple lane', - 'buerbarg', - 'burbarg', - 'first street', - 'frank blvd', - 'frank boulevard', - 'fred road', - 'pear ave', - 'pear avenue' - ], 'confirm map sorted order simulating dawg text order'); - t.deepEqual(map[0], [3, [2, 3]], 'burbarg => [buerbarg, burbarg]'); - t.deepEqual(map[1], [4, [0]], 'first street => 1st st'); - t.deepEqual(map[2], [6, [5]], 'frank boulevard => frank blvd'); - t.deepEqual(map[3], [9, [8]], 'pear avenue => pear ave'); - - cache.writeBatch(map); - - t.deepEqual(map, cache.getAll(), 'dumped contents match input'); - - // test some invalid input - t.throws(() => { cache.writeBatch(); }, 'throws on invalid arguments'); - t.throws(() => { cache.writeBatch(7); }, 'throws on invalid arguments'); - t.throws(() => { cache.writeBatch([7]); }, 'throws on invalid arguments'); - t.throws(() => { cache.writeBatch([[7]]); }, 'throws on invalid arguments'); - t.throws(() => { cache.writeBatch([[7, 'asdf']]); }, 'throws on invalid arguments'); - - return t.end(); -}); - -// tests reading cache from disk -test('read', (t) => { - const cache = new carmenCache.NormalizationCache(file, true); - - t.deepEqual(cache.get(words.indexOf('first street')), [words.indexOf('1st st')], 'normalization value for "first street" is as expected'); - t.equal(cache.get(8888), undefined, 'returns no value for an index not in the cache'); - - const firstWithPrefix = function(p) { - for (let i = 0; i < words.length; i++) if (words[i].startsWith(p)) return i; - }; - - const countWithPrefix = function(p) { - let c = 0; - for (let i = 0; i < words.length; i++) if (words[i].startsWith(p)) c++; - return c; - }; - - t.deepEqual(cache.getPrefixRange(firstWithPrefix('f'), countWithPrefix('f')), [words.indexOf('1st st')], 'found normalization for 1st st but not frank boulevard'); - t.deepEqual(cache.getPrefixRange(firstWithPrefix('frank'), countWithPrefix('frank')), [], 'found nothing because all normalizations share the searched prefix'); - t.deepEqual(cache.getPrefixRange(firstWithPrefix('frank bo'), countWithPrefix('frank bo')), [words.indexOf('frank blvd')], 'found frank boulevard because no prefixes are shared'); - - t.deepEqual(cache.getPrefixRange(firstWithPrefix('bu'), countWithPrefix('bu')), [], 'found nothing because all normalizations share the searched prefix'); - t.deepEqual(cache.getPrefixRange(firstWithPrefix('bue'), countWithPrefix('bue')), [], 'found nothing because bue... doesn\'t normalize to anything'); - t.deepEqual(cache.getPrefixRange(firstWithPrefix('bur'), countWithPrefix('bur')), [words.indexOf('buerbarg')], 'found buerbarg but not burbarg because burbarg shares a prefix with itself'); - t.deepEqual(cache.get(firstWithPrefix('bur')), [words.indexOf('buerbarg'), words.indexOf('burbarg')], 'found buerbarg and burbarg with regular get because nothing gets filtered'); - - // test some invalid input - t.throws(() => { new carmenCache.NormalizationCache(); }, 'throws on invalid arguments'); - t.throws(() => { new carmenCache.NormalizationCache(7); }, 'throws on invalid arguments'); - t.throws(() => { new carmenCache.NormalizationCache('asdf', 7); }, 'throws on invalid arguments'); - - t.throws(() => { new carmenCache.NormalizationCache('/proc', true); }, 'throws on invalid arguments'); - - - t.throws(() => { cache.get(); }, 'throws on invalid arguments'); - t.throws(() => { cache.getPrefixRange('asdf'); }, 'throws on invalid arguments'); - - t.throws(() => { cache.getPrefixRange(); }, 'throws on invalid arguments'); - t.throws(() => { cache.getPrefixRange('asdf'); }, 'throws on invalid arguments'); - t.throws(() => { cache.getPrefixRange('asdf', 'asdf'); }, 'throws on invalid arguments'); - t.throws(() => { cache.getPrefixRange(1, 'asdf'); }, 'throws on invalid arguments'); - t.throws(() => { cache.getPrefixRange(1, 1, 'asdf'); }, 'throws on invalid arguments'); - t.throws(() => { cache.getPrefixRange(1, 1, 1, 'asdf'); }, 'throws on invalid arguments'); - - t.end(); -});