fix: swap node buffers for uint8arrays (#249)

Upgrades all the datastores to get/put `Uint8Array`s instead of node
`Buffer`s.

BREAKING CHANGES:

- Swaps out node `Buffer`s for `Uint8Array`s

Author: achingbrain

README.md

@@ -36,8 +36,8 @@ This is the implementation of the [IPFS repo spec](https://github.com/ipfs/specs
     - [`Promise<boolean> repo.exists()`](#promiseboolean-repoexists)
     - [`Promise<Boolean> repo.isInitialized()`](#promiseboolean-repoisinitialized)
   - [Repos](#repos)
-    - [`Promise repo.put(key, value:Buffer)`](#promise-repoputkey-valuebuffer)
-    - [`Promise<Buffer> repo.get(key)`](#promisebuffer-repogetkey)
+    - [`Promise repo.put(key, value:Uint8Array)`](#promise-repoputkey-valueuint8array)
+    - [`Promise<Uint8Array> repo.get(key)`](#promiseuint8array-repogetkey)
   - [Blocks](#blocks)
     - [`Promise<Block> repo.blocks.put(block:Block)`](#promiseblock-repoblocksputblockblock)
     - [`AsyncIterator<Block> repo.blocks.putMany(source:AsyncIterable<Block>)`](#asynciteratorblock-repoblocksputmanysourceasynciterableblock)
@@ -214,17 +214,17 @@ The returned promise resolves to `false` if the repo has not been initialized an
 
 Root repo:
 
-#### `Promise repo.put(key, value:Buffer)`
+#### `Promise repo.put(key, value:Uint8Array)`
 
 Put a value at the root of the repo
 
-* `key` can be a buffer, a string or a [Key][]
+* `key` can be a Uint8Array, a string or a [Key][]
 
-#### `Promise<Buffer> repo.get(key)`
+#### `Promise<Uint8Array> repo.get(key)`
 
 Get a value at the root of the repo
 
-* `key` can be a buffer, a string or a [Key][]
+* `key` can be a Uint8Array, a string or a [Key][]
 
 ### Blocks
 

package.json

@@ -52,33 +52,33 @@
     "it-first": "^1.0.2",
     "just-range": "^2.1.0",
     "memdown": "^5.1.0",
-    "multihashing-async": "^1.0.0",
+    "multihashing-async": "^2.0.0",
     "ncp": "^2.0.0",
     "rimraf": "^3.0.0",
     "sinon": "^9.0.2"
   },
   "dependencies": {
     "bignumber.js": "^9.0.0",
-    "buffer": "^5.6.0",
     "bytes": "^3.1.0",
-    "cids": "^0.8.0",
-    "datastore-core": "^1.1.0",
-    "datastore-fs": "^1.1.0",
-    "datastore-level": "^1.1.0",
+    "cids": "^1.0.0",
+    "datastore-core": "^2.0.0",
+    "datastore-fs": "^2.0.0",
+    "datastore-level": "^2.0.0",
     "debug": "^4.1.0",
     "err-code": "^2.0.0",
-    "interface-datastore": "^1.0.2",
-    "ipfs-repo-migrations": "^2.0.0",
-    "ipfs-utils": "^2.2.0",
-    "ipld-block": "^0.9.1",
+    "interface-datastore": "^2.0.0",
+    "ipfs-repo-migrations": "^3.0.0",
+    "ipfs-utils": "^2.3.1",
+    "ipld-block": "^0.10.0",
     "it-map": "^1.0.2",
     "it-pushable": "^1.4.0",
     "just-safe-get": "^2.0.0",
     "just-safe-set": "^2.1.0",
-    "multibase": "^1.0.1",
+    "multibase": "^3.0.0",
     "p-queue": "^6.0.0",
     "proper-lockfile": "^4.0.0",
-    "sort-keys": "^4.0.0"
+    "sort-keys": "^4.0.0",
+    "uint8arrays": "^1.0.0"
   },
   "license": "MIT",
   "contributors": [

src/api-addr.js

@@ -1,7 +1,7 @@
 'use strict'
 
-const { Buffer } = require('buffer')
 const Key = require('interface-datastore').Key
+const uint8ArrayFromString = require('uint8arrays/from-string')
 
 const apiFile = new Key('api')
 
@@ -23,7 +23,7 @@ module.exports = (store) => {
      * @returns {Promise<?>}
      */
     async set (value) { // eslint-disable-line require-await
-      return store.put(apiFile, Buffer.from(value.toString()))
+      return store.put(apiFile, uint8ArrayFromString(value.toString()))
     },
     /**
      * Deletes api file

src/blockstore-utils.js

@@ -4,6 +4,7 @@ const { Key } = require('interface-datastore')
 const CID = require('cids')
 const multibase = require('multibase')
 const errcode = require('err-code')
+const uint8ArrayToString = require('uint8arrays/to-string')
 
 /**
  * Transform a cid to the appropriate datastore key.
@@ -16,7 +17,7 @@ exports.cidToKey = cid => {
     throw errcode(new Error('Not a valid cid'), 'ERR_INVALID_CID')
   }
 
-  return new Key('/' + multibase.encode('base32', cid.multihash).toString().slice(1).toUpperCase(), false)
+  return new Key('/' + uint8ArrayToString(multibase.encode('base32', cid.multihash)).slice(1).toUpperCase(), false)
 }
 
 /**

src/config.js

@@ -1,12 +1,13 @@
 'use strict'
 
-const { Buffer } = require('buffer')
 const Key = require('interface-datastore').Key
 const { default: Queue } = require('p-queue')
 const _get = require('just-safe-get')
 const _set = require('just-safe-set')
 const errcode = require('err-code')
 const errors = require('./errors')
+const uint8ArrayToString = require('uint8arrays/to-string')
+const uint8ArrayFromString = require('uint8arrays/from-string')
 
 const configKey = new Key('config')
 
@@ -44,7 +45,7 @@ module.exports = (store) => {
         return
       }
 
-      const config = JSON.parse(encodedValue.toString())
+      const config = JSON.parse(uint8ArrayToString(encodedValue))
       if (key !== undefined && _get(config, key) === undefined) {
         throw new errors.NotFoundError(`Key ${key} does not exist in config`)
       }
@@ -70,7 +71,7 @@ module.exports = (store) => {
         throw errcode(new Error('Invalid key type: ' + typeof key), 'ERR_INVALID_KEY')
       }
 
-      if (value === undefined || Buffer.isBuffer(value)) {
+      if (value === undefined || (value instanceof Uint8Array)) {
         throw errcode(new Error('Invalid value type: ' + typeof value), 'ERR_INVALID_VALUE')
       }
 
@@ -89,7 +90,7 @@ module.exports = (store) => {
      * @returns {void}
      */
     async replace (value, options = {}) { // eslint-disable-line require-await
-      if (!value || Buffer.isBuffer(value)) {
+      if (!value || (value instanceof Uint8Array)) {
         throw errcode(new Error('Invalid value type: ' + typeof value), 'ERR_INVALID_VALUE')
       }
 
@@ -127,7 +128,7 @@ module.exports = (store) => {
   }
 
   function _saveAll (config) {
-    const buf = Buffer.from(JSON.stringify(config, null, 2))
+    const buf = uint8ArrayFromString(JSON.stringify(config, null, 2))
     return store.put(configKey, buf)
   }
 }

src/index.js

@@ -358,7 +358,7 @@ class IpfsRepo {
       count = count.plus(1)
       size = size
         .plus(block.data.byteLength)
-        .plus(block.cid.buffer.byteLength)
+        .plus(block.cid.bytes.byteLength)
     }
 
     return { count, size }
@@ -369,7 +369,7 @@ async function getSize (queryFn) {
   const sum = new Big(0)
   for await (const block of queryFn.query({})) {
     sum.plus(block.value.byteLength)
-      .plus(block.key.toBuffer().byteLength)
+      .plus(block.key.uint8Array().byteLength)
   }
   return sum
 }

src/spec.js

@@ -1,8 +1,9 @@
 'use strict'
 
-const { Buffer } = require('buffer')
 const Key = require('interface-datastore').Key
 const sortKeys = require('sort-keys')
+const uint8ArrayToString = require('uint8arrays/to-string')
+const uint8ArrayFromString = require('uint8arrays/from-string')
 
 const specKey = new Key('datastore_spec')
 
@@ -19,11 +20,11 @@ module.exports = (store) => {
     /**
      * Get the current datastore spec.
      *
-     * @returns {Promise<Buffer>}
+     * @returns {Promise<Uint8Array>}
      */
     async get () {
       const buf = await store.get(specKey)
-      return JSON.parse(buf.toString())
+      return JSON.parse(uint8ArrayToString(buf))
     },
     /**
      * Set the datastore spec of the repo, writing it to the underlying store.
@@ -32,7 +33,7 @@ module.exports = (store) => {
      * @returns {Promise<void>}
      */
     async set (spec) { // eslint-disable-line require-await
-      return store.put(specKey, Buffer.from(JSON.stringify(sortKeys(spec, { deep: true }))))
+      return store.put(specKey, uint8ArrayFromString(JSON.stringify(sortKeys(spec, { deep: true }))))
     }
   }
 }

src/version.js

@@ -1,9 +1,10 @@
 'use strict'
 
-const { Buffer } = require('buffer')
 const Key = require('interface-datastore').Key
 const debug = require('debug')
 const log = debug('ipfs:repo:version')
+const uint8ArrayToString = require('uint8arrays/to-string')
+const uint8ArrayFromString = require('uint8arrays/from-string')
 
 const versionKey = new Key('version')
 
@@ -24,7 +25,7 @@ module.exports = (store) => {
      */
     async get () {
       const buf = await store.get(versionKey)
-      return parseInt(buf.toString().trim(), 10)
+      return parseInt(uint8ArrayToString(buf), 10)
     },
     /**
      * Set the version of the repo, writing it to the underlying store.
@@ -33,7 +34,7 @@ module.exports = (store) => {
      * @returns {Promise<void>}
      */
     async set (version) { // eslint-disable-line require-await
-      return store.put(versionKey, Buffer.from(String(version)))
+      return store.put(versionKey, uint8ArrayFromString(String(version)))
     },
     /**
      * Check the current version, and returns true if versions matches

test/api-addr-test.js

@@ -1,9 +1,9 @@
 /* eslint-env mocha */
 'use strict'
 
-const { Buffer } = require('buffer')
 const { expect } = require('./utils/chai')
 const apiAddr = require('../src/api-addr')
+const uint8ArrayFromString = require('uint8arrays/from-string')
 
 module.exports = () => {
   describe('api-addr', () => {
@@ -31,7 +31,7 @@ module.exports = () => {
 
         await api.set('0')
 
-        expect(val).to.deep.equal(Buffer.from('0'))
+        expect(val).to.deep.equal(uint8ArrayFromString('0'))
       })
     })
   })