Merge pull request #97 from ssb-ngi-pointer/prefix-offset

Add prefixOffset option

Author: arj03

README.md

@@ -390,6 +390,11 @@ inverted index that allows for faster queries at the cost of extra
 space. Maps don't store empty values meaning they are also a good fit
 for sparce indexes such as vote links.
 
+It is possible specifiy where in the target the prefix buffer should
+be constructed from using `prefixOffset`. This is useful for targets
+that starts with a common prefix such as % in order to increase the
+information amount.
+
 ## Low-level API
 
 First some terminology: offset refers to the byte position in the log

index.js

@@ -302,13 +302,13 @@ module.exports = function (log, indexesPath) {
     }
   }
 
-  function safeReadUint32(buf) {
+  function safeReadUint32(buf, prefixOffset = 0) {
     if (buf.length < 4) {
       const bigger = Buffer.alloc(4)
       buf.copy(bigger)
-      return bigger.readUInt32LE(0)
+      return bigger.readUInt32LE(prefixOffset)
     } else {
-      return buf.readUInt32LE(0)
+      return buf.readUInt32LE(prefixOffset)
     }
   }
 
@@ -324,7 +324,11 @@ module.exports = function (log, indexesPath) {
       const fieldStart = opData.seek(buffer)
       if (~fieldStart) {
         const buf = bipf.slice(buffer, fieldStart)
-        addToPrefixMap(index.map, seq, buf.length ? safeReadUint32(buf) : 0)
+        addToPrefixMap(
+          index.map,
+          seq,
+          buf.length ? safeReadUint32(buf, opData.prefixOffset) : 0
+        )
       }
 
       index.offset = offset
@@ -339,7 +343,9 @@ module.exports = function (log, indexesPath) {
       const fieldStart = opData.seek(buffer)
       if (~fieldStart) {
         const buf = bipf.slice(buffer, fieldStart)
-        index.tarr[seq] = buf.length ? safeReadUint32(buf) : 0
+        index.tarr[seq] = buf.length
+          ? safeReadUint32(buf, opData.prefixOffset)
+          : 0
       } else {
         index.tarr[seq] = 0
       }
@@ -683,7 +689,9 @@ module.exports = function (log, indexesPath) {
 
   function matchAgainstPrefix(op, prefixIndex, cb) {
     const target = op.data.value
-    const targetPrefix = target ? safeReadUint32(target) : 0
+    const targetPrefix = target
+      ? safeReadUint32(target, op.data.prefixOffset)
+      : 0
     const bitset = new TypedFastBitSet()
     const done = multicb({ pluck: 1 })
 

test/helpers.js

@@ -1,5 +1,6 @@
 const bipf = require('bipf')
 
+const bKey = Buffer.from('key')
 const bValue = Buffer.from('value')
 const bVote = Buffer.from('vote')
 const bLink = Buffer.from('link')
@@ -14,6 +15,11 @@ const bPrivate = Buffer.from('private')
 const bChannel = Buffer.from('channel')
 
 module.exports = {
+  seekKey: function (buffer) {
+    var p = 0 // note you pass in p!
+    return bipf.seekKey(buffer, p, bKey)
+  },
+
   seekAuthor: function (buffer) {
     var p = 0 // note you pass in p!
     p = bipf.seekKey(buffer, p, bValue)

test/prefix.js

@@ -375,3 +375,39 @@ prepareAndRunTest('Prefix map equal', dir, (t, db, raf) => {
     })
   })
 })
+
+prepareAndRunTest('Prefix offset', dir, (t, db, raf) => {
+  const msg1 = { type: 'post', text: 'Testing!' }
+  const msg2 = { type: 'contact', text: 'Testing!' }
+  const msg3 = { type: 'post', text: 'Testing 2!' }
+
+  let state = validate.initial()
+  state = validate.appendNew(state, null, keys, msg1, Date.now())
+  state = validate.appendNew(state, null, keys, msg2, Date.now() + 1)
+  state = validate.appendNew(state, null, keys, msg3, Date.now() + 2)
+
+  addMsg(state.queue[0].value, raf, (err, msg) => {
+    addMsg(state.queue[1].value, raf, (err, msg) => {
+      addMsg(state.queue[2].value, raf, (err, msg) => {
+        const typeQuery = {
+          type: 'EQUAL',
+          data: {
+            seek: helpers.seekKey,
+            value: Buffer.from(msg.key),
+            indexType: 'key',
+            indexName: 'value_key_' + msg.key,
+            useMap: true,
+            prefix: 32,
+            prefixOffset: 1,
+          },
+        }
+
+        db.all(typeQuery, 0, false, false, (err, results) => {
+          t.equal(results.length, 1)
+          t.equal(results[0].value.content.text, 'Testing 2!')
+          t.end()
+        })
+      })
+    })
+  })
+})