feat: ipns over dht

Author: vasco-santos

package.json

@@ -118,17 +118,17 @@
     "ipld-ethereum": "^2.0.1",
     "ipld-git": "~0.2.2",
     "ipld-zcash": "~0.1.6",
-    "ipns": "~0.3.0",
+    "ipns": "~0.4.2",
     "is-ipfs": "~0.4.7",
     "is-pull-stream": "~0.0.0",
     "is-stream": "^1.1.0",
     "joi": "^13.4.0",
     "joi-browser": "^13.4.0",
     "joi-multiaddr": "^3.0.0",
-    "libp2p": "~0.24.0",
+    "libp2p": "libp2p/js-libp2p#master",
     "libp2p-bootstrap": "~0.9.3",
     "libp2p-crypto": "~0.14.1",
-    "libp2p-kad-dht": "~0.11.1",
+    "libp2p-kad-dht": "libp2p/js-libp2p-kad-dht#feat/allow-configurable-validators-and-selectors",
     "libp2p-keychain": "~0.3.3",
     "libp2p-mdns": "~0.12.0",
     "libp2p-mplex": "~0.8.4",

src/core/components/libp2p.js

@@ -3,6 +3,7 @@
 const promisify = require('promisify-es6')
 const get = require('lodash/get')
 const defaultsDeep = require('@nodeutils/defaults-deep')
+const ipnsUtils = require('../ipns/routing/utils')
 
 module.exports = function libp2p (self) {
   return {
@@ -16,6 +17,7 @@ module.exports = function libp2p (self) {
 
         const defaultBundle = (opts) => {
           const libp2pDefaults = {
+            datastore: opts.datastore,
             peerInfo: opts.peerInfo,
             peerBook: opts.peerBook,
             config: {
@@ -43,6 +45,14 @@ module.exports = function libp2p (self) {
                     get(opts.config, 'relay.hop.active', false))
                 }
               },
+              dht: {
+                validators: {
+                  ipns: ipnsUtils.validator
+                },
+                selectors: {
+                  ipns: ipnsUtils.selector
+                }
+              },
               EXPERIMENTAL: {
                 dht: get(opts.options, 'EXPERIMENTAL.dht', false),
                 pubsub: get(opts.options, 'EXPERIMENTAL.pubsub', false)
@@ -72,6 +82,7 @@ module.exports = function libp2p (self) {
         self._libp2pNode = libp2pBundle({
           options: self._options,
           config: config,
+          datastore: self._repo.datastore,
           peerInfo: self._peerInfo,
           peerBook: self._peerInfoBook
         })

src/core/components/start.js

@@ -2,12 +2,14 @@
 
 const series = require('async/series')
 const Bitswap = require('ipfs-bitswap')
+const get = require('lodash/get')
 const setImmediate = require('async/setImmediate')
 const promisify = require('promisify-es6')
 const { TieredDatastore } = require('datastore-core')
 
 const IPNS = require('../ipns')
 const OfflineDatastore = require('../ipns/routing/offline-datastore')
+const DhtDatastore = require('../ipns/routing/dht-datastore')
 
 module.exports = (self) => {
   return promisify((callback) => {
@@ -43,10 +45,15 @@ module.exports = (self) => {
 
         // TODO Add IPNS pubsub if enabled
 
-        // NOTE: IPNS routing is being replaced by the local repo datastore while the IPNS over DHT is not ready
-        // When DHT is added, if local option enabled, should receive offlineDatastore as well
-        const offlineDatastore = new OfflineDatastore(self._repo)
-        ipnsStores.push(offlineDatastore)
+        // DHT should be added as routing if we are not running with local flag
+        // TODO: Need to change this logic once DHT is enabled by default, for now fallback to Offline datastore
+        if (get(self._options, 'EXPERIMENTAL.dht', false) && !self._options.local) {
+          const dhtDatastore = new DhtDatastore(self._libp2pNode.dht)
+          ipnsStores.push(dhtDatastore)
+        } else {
+          const offlineDatastore = new OfflineDatastore(self._repo)
+          ipnsStores.push(offlineDatastore)
+        }
 
         // Create ipns routing with a set of datastores
         const routing = new TieredDatastore(ipnsStores)

src/core/ipns/publisher.js

@@ -1,7 +1,6 @@
 'use strict'
 
 const PeerId = require('peer-id')
-const Record = require('libp2p-record').Record
 const { Key } = require('interface-datastore')
 const series = require('async/series')
 const errcode = require('err-code')
@@ -57,7 +56,6 @@ class IpnsPublisher {
       log.error(errMsg)
       return callback(errcode(new Error(errMsg), 'ERR_INVALID_PEER_ID'))
     }
-
     const publicKey = peerId._pubKey
 
     ipns.embedPublicKey(publicKey, record, (err, embedPublicKeyRecord) => {
@@ -97,19 +95,17 @@ class IpnsPublisher {
       return callback(errcode(new Error(errMsg), 'ERR_INVALID_DATASTORE_KEY'))
     }
 
-    let rec
+    let entryData
     try {
       // Marshal record
-      const entryData = ipns.marshal(entry)
-      // Marshal to libp2p record
-      rec = new Record(key.toBuffer(), entryData)
+      entryData = ipns.marshal(entry)
     } catch (err) {
       log.error(err)
       return callback(err)
     }
 
     // Add record to routing (buffer key)
-    this._routing.put(key.toBuffer(), rec.serialize(), (err, res) => {
+    this._routing.put(key.toBuffer(), entryData, (err, res) => {
       if (err) {
         const errMsg = `ipns record for ${key.toString()} could not be stored in the routing`
 
@@ -137,17 +133,8 @@ class IpnsPublisher {
       return callback(errcode(new Error(errMsg), 'ERR_UNDEFINED_PARAMETER'))
     }
 
-    let rec
-    try {
-      // Marshal to libp2p record
-      rec = new Record(key.toBuffer(), publicKey.bytes)
-    } catch (err) {
-      log.error(err)
-      return callback(err)
-    }
-
     // Add public key to routing (buffer key)
-    this._routing.put(key.toBuffer(), rec.serialize(), (err, res) => {
+    this._routing.put(key.toBuffer(), publicKey.bytes, (err, res) => {
       if (err) {
         const errMsg = `public key for ${key.toString()} could not be stored in the routing`
 
@@ -174,45 +161,55 @@ class IpnsPublisher {
     const checkRouting = !(options.checkRouting === false)
 
     this._repo.datastore.get(ipns.getLocalKey(peerId.id), (err, dsVal) => {
-      let result
-
       if (err) {
         if (err.code !== 'ERR_NOT_FOUND') {
           const errMsg = `unexpected error getting the ipns record ${peerId.id} from datastore`
 
           log.error(errMsg)
           return callback(errcode(new Error(errMsg), 'ERR_UNEXPECTED_DATASTORE_RESPONSE'))
-        } else {
-          if (!checkRouting) {
+        }
+
+        if (!checkRouting) {
+          return callback(null, null)
+        }
+
+        // Try to get from routing
+        let keys
+        try {
+          keys = ipns.getIdKeys(peerId.toBytes())
+        } catch (err) {
+          log.error(err)
+          return callback(err)
+        }
+
+        this._routing.get(keys.routingKey, (err, res) => {
+          if (err) {
+            log(`error when determining the last published IPNS record for ${peerId.id}`)
             return callback(null, null)
-          } else {
-            // TODO ROUTING - get from DHT
-            return callback(new Error('not implemented yet'))
           }
-        }
-      }
 
-      if (Buffer.isBuffer(dsVal)) {
-        result = dsVal
+          // unmarshal data
+          this._unmarshalData(res, callback)
+        })
       } else {
-        const errMsg = `found ipns record that we couldn't convert to a value`
-
-        log.error(errMsg)
-        return callback(errcode(new Error(errMsg), 'ERR_INVALID_IPNS_RECORD'))
+        // unmarshal data
+        this._unmarshalData(dsVal, callback)
       }
+    })
+  }
 
-      // unmarshal data
-      try {
-        result = ipns.unmarshal(dsVal)
-      } catch (err) {
-        const errMsg = `found ipns record that we couldn't convert to a value`
+  _unmarshalData (data, callback) {
+    let result
+    try {
+      result = ipns.unmarshal(data)
+    } catch (err) {
+      const errMsg = `found ipns record that we couldn't convert to a value`
 
-        log.error(errMsg)
-        return callback(null, null)
-      }
+      log.error(errMsg)
+      return callback(null, null)
+    }
 
-      callback(null, result)
-    })
+    callback(null, result)
   }
 
   _updateOrCreateRecord (privKey, value, validity, peerId, callback) {
@@ -224,7 +221,7 @@ class IpnsPublisher {
     }
 
     const getPublishedOptions = {
-      checkRouting: false // TODO ROUTING - change to true
+      checkRouting: true
     }
 
     this._getPublished(peerId, getPublishedOptions, (err, record) => {

src/core/ipns/resolver.js

@@ -1,9 +1,10 @@
 'use strict'
 
 const ipns = require('ipns')
-const Record = require('libp2p-record').Record
+const crypto = require('libp2p-crypto')
 const PeerId = require('peer-id')
 const errcode = require('err-code')
+const parallel = require('async/parallel')
 
 const debug = require('debug')
 const log = debug('jsipfs:ipns:resolver')
@@ -97,13 +98,14 @@ class IpnsResolver {
       return callback(err)
     }
 
-    const { routingKey } = ipns.getIdKeys(peerId.toBytes())
+    const { routingKey, routingPubKey } = ipns.getIdKeys(peerId.toBytes())
 
-    // TODO DHT - get public key from routing?
-    // https://github.com/ipfs/go-ipfs/blob/master/namesys/routing.go#L70
-    // https://github.com/libp2p/go-libp2p-routing/blob/master/routing.go#L99
-
-    this._routing.get(routingKey.toBuffer(), (err, res) => {
+    parallel([
+      // Name should be the hash of a public key retrievable from ipfs.
+      // We retrieve public key to add it to the PeerId, as the IPNS record may not have it.
+      (cb) => this._routing.get(routingPubKey.toBuffer(), cb),
+      (cb) => this._routing.get(routingKey.toBuffer(), cb)
+    ], (err, res) => {
       if (err) {
         if (err.code !== 'ERR_NOT_FOUND') {
           const errMsg = `unexpected error getting the ipns record ${peerId.id}`
@@ -117,10 +119,21 @@ class IpnsResolver {
         return callback(errcode(new Error(errMsg), 'ERR_NO_RECORD_FOUND'))
       }
 
+      // Public key
+      try {
+        // Insert it into the peer id public key, to be validated by IPNS validator
+        peerId.pubKey = crypto.keys.unmarshalPublicKey(res[0])
+      } catch (err) {
+        const errMsg = `found public key record that we couldn't convert to a value`
+
+        log.error(errMsg)
+        return callback(errcode(new Error(errMsg), 'ERR_INVALID_PUB_KEY_RECEIVED'))
+      }
+
+      // IPNS entry
       let ipnsEntry
       try {
-        const record = Record.deserialize(res)
-        ipnsEntry = ipns.unmarshal(record.value)
+        ipnsEntry = ipns.unmarshal(res[1])
       } catch (err) {
         const errMsg = `found ipns record that we couldn't convert to a value`
 

src/core/ipns/routing/dht-datastore.js

@@ -0,0 +1,42 @@
+'use strict'
+
+// const { Key } = require('interface-datastore')
+// const { encodeBase32 } = require('./utils')
+
+// Dht datastore sets the proper encoding for storing records
+class DhtDatastore {
+  constructor (dht) {
+    this._dht = dht
+  }
+
+  /**
+   * Put a value to the dht indexed by the received key properly encoded.
+   * @param {Buffer} key identifier of the value.
+   * @param {Buffer} value value to be stored.
+   * @param {function(Error)} callback
+   * @returns {void}
+   */
+  put (key, value, callback) {
+    // encode key properly - base32(/ipns/{cid}) TODO: remove this
+    // const routingKey = new Key('/' + encodeBase32(key), false)
+
+    this._dht.put(key, value, callback)
+    // this._dht.put(routingKey.toBuffer(), value, callback)
+  }
+
+  /**
+   * Get a value from the local datastore indexed by the received key properly encoded.
+   * @param {Buffer} key identifier of the value to be obtained.
+   * @param {function(Error, Buffer)} callback
+   * @returns {void}
+   */
+  get (key, callback) {
+    // encode key properly - base32(/ipns/{cid}) TODO: remove this
+    // const routingKey = new Key('/' + encodeBase32(key), false)
+
+    this._dht.get(key, callback)
+    // this._dht.get(routingKey.toBuffer(), callback)
+  }
+}
+
+exports = module.exports = DhtDatastore

src/core/ipns/routing/offline-datastore.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const { Key } = require('interface-datastore')
+const Record = require('libp2p-record').Record
 const { encodeBase32 } = require('./utils')
 
 const errcode = require('err-code')
@@ -48,7 +49,10 @@ class OfflineDatastore {
       return callback(errcode(new Error(errMsg), 'ERR_GENERATING_ROUTING_KEY'))
     }
 
-    this._repo.datastore.put(routingKey, value, callback)
+    // Marshal to libp2p record as the DHT does
+    let record = new Record(key, value)
+
+    this._repo.datastore.put(routingKey, record.serialize(), callback)
   }
 
   /**
@@ -76,7 +80,22 @@ class OfflineDatastore {
       return callback(errcode(new Error(errMsg), 'ERR_GENERATING_ROUTING_KEY'))
     }
 
-    this._repo.datastore.get(routingKey, callback)
+    this._repo.datastore.get(routingKey, (err, res) => {
+      if (err) {
+        return callback(err)
+      }
+
+      // Unmarshal libp2p record as the DHT does
+      let record
+      try {
+        record = Record.deserialize(res)
+      } catch (err) {
+        log.error(err)
+        return callback(err)
+      }
+
+      callback(null, record.value)
+    })
   }
 
   // encode key properly - base32(/ipns/{cid})