feat: integration of js-ipfs-repo-migrations

Integration of js-ipfs-repo-migrations brings automatic repo migrations
to ipfs-repo (both in-browser and fs). It is possible to control the
automatic migration using either config's setting
'repoDisableAutoMigration' or IPFSRepo's option 'disableAutoMigration'.

License: MIT
Signed-off-by: Adam Uhlir <[email protected]>

Author: AuHau

README.md

@@ -28,6 +28,7 @@ This is the implementation of the [IPFS repo spec](https://github.com/ipfs/specs
   - [Use in a browser Using a script tag](#use-in-a-browser-using-a-script-tag)
 - [Usage](#usage)
 - [API](#api)
+- [Notes](#notes)
 - [Contribute](#contribute)
 - [License](#license)
 
@@ -318,6 +319,11 @@ Returned promise resolves to a `boolean` indicating the existence of the lock.
 
 - [Explanation of how repo is structured](https://github.com/ipfs/js-ipfs-repo/pull/111#issuecomment-279948247)
 
+### Migrations
+
+When there is a new repo migration and the version of repo is increased, don't 
+forget to propagate the changes into the test repo (`test/test-repo`).
+
 ## Contribute
 
 There are some ways you can make this module better:

package.json

@@ -51,7 +51,8 @@
     "multihashes": "~0.4.14",
     "multihashing-async": "~0.7.0",
     "ncp": "^2.0.0",
-    "rimraf": "^2.6.3"
+    "rimraf": "^2.6.3",
+    "sinon": "^7.3.1"
   },
   "dependencies": {
     "base32.js": "~0.1.0",
@@ -64,6 +65,7 @@
     "err-code": "^1.1.2",
     "interface-datastore": "~0.7.0",
     "ipfs-block": "~0.8.1",
+    "ipfs-repo-migrations": "AuHau/js-ipfs-repo-migrations#dev",
     "just-safe-get": "^1.3.0",
     "just-safe-set": "^2.1.0",
     "lodash.has": "^4.5.2",
@@ -73,6 +75,7 @@
   },
   "license": "MIT",
   "contributors": [
+    "Adam Uhlir<[email protected]>",
     "Alan Shaw <[email protected]>",
     "Alex Potsides <[email protected]>",
     "Brian Hoffman <[email protected]>",

src/default-options-browser.js

@@ -2,6 +2,7 @@
 
 // Default configuration for a repo in the browser
 module.exports = {
+  disableAutoMigration: false,
   lock: 'memory',
   storageBackends: {
     root: require('datastore-level'),

src/default-options.js

@@ -2,6 +2,7 @@
 
 // Default configuration for a repo in node.js
 module.exports = {
+  disableAutoMigration: false,
   lock: 'fs',
   storageBackends: {
     root: require('datastore-fs'),

src/errors/index.js

@@ -15,6 +15,21 @@ class NotFoundError extends Error {
 NotFoundError.code = 'ERR_NOT_FOUND'
 exports.NotFoundError = NotFoundError
 
+/**
+ * Error raised when version of the stored repo is not compatible with version of this package.
+ */
+class InvalidRepoVersionError extends Error {
+  constructor (message) {
+    super(message)
+    this.name = 'InvalidRepoVersionError'
+    this.code = 'ERR_INVALID_REPO_VERSION'
+    this.message = message
+  }
+}
+
+InvalidRepoVersionError.code = 'ERR_INVALID_REPO_VERSION'
+exports.InvalidRepoVersionError = InvalidRepoVersionError
+
 exports.ERR_REPO_NOT_INITIALIZED = 'ERR_REPO_NOT_INITIALIZED'
 exports.ERR_REPO_ALREADY_OPEN = 'ERR_REPO_ALREADY_OPEN'
 exports.ERR_REPO_ALREADY_CLOSED = 'ERR_REPO_ALREADY_CLOSED'

src/index.js

@@ -6,7 +6,9 @@ const path = require('path')
 const debug = require('debug')
 const Big = require('bignumber.js')
 const errcode = require('err-code')
+const migrator = require('ipfs-repo-migrations')
 
+const constants = require('./constants')
 const backends = require('./backends')
 const version = require('./version')
 const config = require('./config')
@@ -26,8 +28,6 @@ const lockers = {
   fs: require('./lock')
 }
 
-const repoVersion = require('./constants').repoVersion
-
 /**
  * IpfsRepo implements all required functionality to read and write to an ipfs repo.
  *
@@ -64,7 +64,7 @@ class IpfsRepo {
     await this._openRoot()
     await this.config.set(buildConfig(config))
     await this.spec.set(buildDatastoreSpec(config))
-    await this.version.set(repoVersion)
+    await this.version.set(constants.repoVersion)
   }
 
   /**
@@ -92,6 +92,17 @@ class IpfsRepo {
       this.blocks = await blockstore(blocksBaseStore, this.options.storageBackendOptions.blocks)
       log('creating keystore')
       this.keys = backends.create('keys', path.join(this.path, 'keys'), this.options)
+
+      if (!await this.version.check(constants.repoVersion)) {
+        log('Something is fishy')
+        if (!this.options.disableAutoMigration) {
+          log('Let see what')
+          await this._migrate(constants.repoVersion)
+        } else {
+          throw new ERRORS.InvalidRepoVersionError('Incompatible repo versions. Automatic migrations disabled. Please migrate the repo manually.')
+        }
+      }
+
       this.closed = false
       log('all opened')
     } catch (err) {
@@ -176,7 +187,7 @@ class IpfsRepo {
       [config] = await Promise.all([
         this.config.exists(),
         this.spec.exists(),
-        this.version.check(repoVersion)
+        this.version.exists()
       ])
     } catch (err) {
       if (err.code === 'ERR_NOT_FOUND') {
@@ -264,6 +275,40 @@ class IpfsRepo {
     }
   }
 
+  async _migrate (toVersion) {
+    let disableMigrationsConfig
+    try {
+      disableMigrationsConfig = await this.config.get('repoDisableAutoMigration')
+    } catch (e) {
+      if (e.code === ERRORS.NotFoundError.code) {
+        disableMigrationsConfig = false
+      } else {
+        throw e
+      }
+    }
+
+    if (disableMigrationsConfig) {
+      throw new ERRORS.InvalidRepoVersionError('Incompatible repo versions. Automatic migrations disabled. Please migrate the repo manually.')
+    }
+
+    const currentRepoVersion = await this.version.get()
+    log(currentRepoVersion)
+    if (currentRepoVersion >= toVersion) {
+      if (currentRepoVersion > toVersion) {
+        log('Your repo\'s version is higher then this version of js-ipfs-repo require! You should revert it.')
+      }
+
+      log('Nothing to migrate')
+      return
+    }
+
+    if (toVersion > migrator.getLatestMigrationVersion()) {
+      throw new Error('The ipfs-repo-migrations package does not have migration for version: ' + toVersion)
+    }
+
+    return migrator.migrate(this.path, { toVersion: toVersion, ignoreLock: true, repoOptions: this.options })
+  }
+
   async _storageMaxStat () {
     try {
       const max = await this.config.get('Datastore.StorageMax')
@@ -298,7 +343,7 @@ async function getSize (queryFn) {
 }
 
 module.exports = IpfsRepo
-module.exports.repoVersion = repoVersion
+module.exports.repoVersion = constants.repoVersion
 module.exports.errors = ERRORS
 
 function buildOptions (_options) {

src/version.js

@@ -3,7 +3,6 @@
 const Key = require('interface-datastore').Key
 const debug = require('debug')
 const log = debug('repo:version')
-const errcode = require('err-code')
 
 const versionKey = new Key('version')
 
@@ -36,9 +35,9 @@ module.exports = (store) => {
       return store.put(versionKey, Buffer.from(String(version)))
     },
     /**
-     * Check the current version, and return an error on missmatch
+     * Check the current version, and returns true if versions matches
      * @param {number} expected
-     * @returns {void}
+     * @returns {boolean}
      */
     async check (expected) {
       const version = await this.get()
@@ -47,9 +46,7 @@ module.exports = (store) => {
       // TODO: Clean up the compatibility logic. Repo feature detection would be ideal, or a better version schema
       const compatibleVersion = (version === 6 && expected === 7) || (expected === 6 && version === 7)
 
-      if (version !== expected && !compatibleVersion) {
-        throw errcode(new Error(`ipfs repo needs migration: expected version v${expected}, found version v${version}`), 'ERR_INVALID_REPO_VERSION')
-      }
+      return version === expected || compatibleVersion
     }
   }
 }

test/browser.js

@@ -4,8 +4,29 @@
 
 const IPFSRepo = require('../src')
 
+async function createTempRepo ({ dontOpen, opts }) {
+  const date = Date.now().toString()
+  const repoPath = 'test-repo-for-' + date
+
+  const repo = new IPFSRepo(repoPath, opts)
+  await repo.init({})
+
+  if (!dontOpen) {
+    await repo.open()
+  }
+
+  return {
+    path: repoPath,
+    instance: repo,
+    teardown: async () => {
+    }
+  }
+}
+
 describe('IPFS Repo Tests on the Browser', () => {
   require('./options-test')
+  require('./migrations-test')(createTempRepo)
+
   const repo = new IPFSRepo('myrepo')
 
   before(async () => {

test/migrations-test.js

@@ -0,0 +1,146 @@
+/* eslint max-nested-callbacks: ["error", 8] */
+/* eslint-env mocha */
+'use strict'
+
+const chai = require('chai')
+const dirtyChai = require('dirty-chai')
+const expect = chai.expect
+chai.use(dirtyChai)
+const sinon = require('sinon')
+
+const migrator = require('ipfs-repo-migrations')
+const constants = require('../src/constants')
+const errors = require('../src/errors')
+const IPFSRepo = require('../src')
+
+module.exports = (createTempRepo) => {
+  describe('Migrations tests', () => {
+    let teardown
+    let repo
+    let migrateStub
+    let repoVersionStub
+    let getLatestMigrationVersionStub
+
+    before(() => {
+      repoVersionStub = sinon.stub(constants, 'repoVersion')
+      migrateStub = sinon.stub(migrator, 'migrate')
+      getLatestMigrationVersionStub = sinon.stub(migrator, 'getLatestMigrationVersion')
+    })
+
+    after(() => {
+      repoVersionStub.restore()
+      migrateStub.restore()
+      getLatestMigrationVersionStub.restore()
+    })
+
+    beforeEach(async () => {
+      ({ instance: repo, teardown } = await createTempRepo({}))
+      sinon.reset()
+    })
+
+    afterEach(async () => {
+      await teardown()
+    })
+
+    it('should migrate by default', async () => {
+      migrateStub.resolves()
+      repoVersionStub.value(8)
+      getLatestMigrationVersionStub.returns(9)
+
+      await repo.version.set(7)
+      await repo.close()
+
+      expect(migrateStub.called).to.be.false()
+
+      await repo.open()
+
+      expect(migrateStub.called).to.be.true()
+    })
+
+    it('should not migrate when option disableAutoMigration is true', async () => {
+      migrateStub.resolves()
+      repoVersionStub.resolves(8)
+      getLatestMigrationVersionStub.returns(9)
+
+      await repo.version.set(7)
+      await repo.close()
+
+      const newOpts = Object.assign({}, repo.options)
+      newOpts.disableAutoMigration = true
+      const newRepo = new IPFSRepo(repo.path, newOpts)
+
+      expect(migrateStub.called).to.be.false()
+      try {
+        await newRepo.open()
+        throw Error('Should throw error')
+      } catch (e) {
+        expect(e.code).to.equal(errors.InvalidRepoVersionError.code)
+      }
+
+      expect(migrateStub.called).to.be.false()
+    })
+
+    it('should not migrate when config option repoDisableAutoMigration is true', async () => {
+      migrateStub.resolves()
+      repoVersionStub.resolves(8)
+      getLatestMigrationVersionStub.returns(9)
+
+      await repo.config.set('repoDisableAutoMigration', true)
+      await repo.version.set(7)
+      await repo.close()
+
+      expect(migrateStub.called).to.be.false()
+      try {
+        await repo.open()
+        throw Error('Should throw error')
+      } catch (e) {
+        expect(migrateStub.called).to.be.false()
+        expect(e.code).to.equal(errors.InvalidRepoVersionError.code)
+      }
+    })
+
+    it('should not migrate when versions matches', async () => {
+      migrateStub.resolves()
+      repoVersionStub.value(8)
+
+      await repo.version.set(8)
+      await repo.close()
+
+      expect(migrateStub.called).to.be.false()
+
+      await repo.open()
+
+      expect(migrateStub.called).to.be.false()
+    })
+
+    it('should not migrate when current repo versions is higher then expected', async () => {
+      migrateStub.resolves()
+      repoVersionStub.value(8)
+
+      await repo.version.set(9)
+      await repo.close()
+
+      expect(migrateStub.called).to.be.false()
+
+      await repo.open()
+
+      expect(migrateStub.called).to.be.false()
+    })
+
+    it('should throw error if ipfs-repo-migrations does not contain expected migration', async () => {
+      migrateStub.resolves()
+      repoVersionStub.value(8)
+      getLatestMigrationVersionStub.returns(7)
+
+      await repo.version.set(7)
+      await repo.close()
+
+      try {
+        await repo.open()
+        throw Error('Should throw')
+      } catch (e) {
+        expect(e.message).to.include('package does not have migration')
+      }
+    })
+  })
+}