Merge pull request #3 from ipfs/cli-integration

feat: integrate with jsipfs cli

README.md

@@ -62,6 +62,18 @@ Loading this module through a script tag will make the `mfs` obj available in th
 <script src="https://npmcdn.com/ipfs-mfs/dist/index.js"></script>
 ```
 
+### A note on concurrency
+
+The mfs works by storing a reference to the root node's CID in LevelDB. LevelDB does not support concurrent access so there are read/write locks around bits of the code that modify the the root node's CID.
+
+A lock is kept on the main thread and any requests to read/write from workers or the main thread itself are queued pending release of the lock by the existing holder.
+
+Reads are executed together, writes are executed sequentially and prevent any reads from starting.
+
+If you are using IPFS in a single process or with the node [cluster](https://nodejs.org/api/cluster.html) module this should be completely transparent.
+
+If you are using [Web Workers](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API) there is no way to globally listen to messages sent between workers and the main thread so you will need to also use the [observable-webworkers](https://www.npmjs.com/package/observable-webworkers) module to ensure the right message transports are set up to allow requesting/releasing the locks.
+
 ## Contribute
 
 All are welcome, please join in!

cli.js

@@ -0,0 +1,3 @@
+'use strict'
+
+module.exports = require('./src/cli')

core.js

@@ -0,0 +1,3 @@
+'use strict'
+
+module.exports = require('./src/core')

http.js

@@ -0,0 +1,3 @@
+'use strict'
+
+module.exports = require('./src/http')

package.json

@@ -10,7 +10,8 @@
   "scripts": {
     "test": "cross-env aegir test -f test/browser.js",
     "test:node": "aegir test -t node",
-    "test:browser": "aegir test -t browser -t webworker -f test/browser.js",
+    "test:browser": "aegir test -t browser -f test/browser.js",
+    "test:webworker": "aegir test -t webworker -f test/browser.js",
     "build": "aegir build",
     "lint": "aegir lint",
     "release": "aegir release",
@@ -38,13 +39,14 @@
   "devDependencies": {
     "@commitlint/cli": "^6.2.0",
     "@commitlint/config-conventional": "^6.1.3",
-    "aegir": "^13.0.6",
+    "aegir": "^14.0.0",
     "chai": "^4.1.2",
     "cross-env": "^5.1.4",
+    "detect-webworker": "^1.0.0",
     "dirty-chai": "^2.0.1",
-    "ipfs": "~0.28.2",
+    "ipfs": "~0.29.0",
     "pre-commit": "^1.2.2",
-    "pre-push": "^0.1.1",
+    "pre-push": "~0.1.1",
     "safe-buffer": "^5.1.1",
     "sign-commit": "~0.1.0",
     "tmp": "~0.0.33"
@@ -59,10 +61,12 @@
     "file-api": "~0.10.4",
     "filereader-stream": "^2.0.0",
     "interface-datastore": "~0.4.2",
-    "ipfs-unixfs": "~0.1.14",
-    "ipfs-unixfs-engine": "~0.29.0",
+    "ipfs-unixfs": "~0.1.15",
+    "ipfs-unixfs-engine": "~0.30.0",
     "is-pull-stream": "~0.0.0",
     "is-stream": "^1.1.0",
+    "joi": "^13.4.0",
+    "mortice": "^1.0.0",
     "promisify-es6": "^1.0.3",
     "pull-cat": "^1.1.11",
     "pull-paramap": "^1.2.2",

src/cli/cp.js

@@ -1,22 +1,33 @@
 'use strict'
 
+const {
+  asBoolean
+} = require('./utils')
+
 module.exports = {
   command: 'cp <source> <dest>',
 
-  describe: 'Copy files between locations in the mfs.',
+  describe: 'Copy files between locations in the mfs',
 
   builder: {
     parents: {
       alias: 'p',
       type: 'boolean',
       default: false,
+      coerce: asBoolean,
       describe: 'Create any non-existent intermediate directories'
     },
-    recursive: {
-      alias: 'r',
-      type: 'boolean',
-      default: false,
-      describe: 'Remove directories recursively'
+    format: {
+      alias: 'h',
+      type: 'string',
+      default: 'dag-pb',
+      describe: 'If intermediate directories are created, use this format to create them (experimental)'
+    },
+    hash: {
+      alias: 'h',
+      type: 'string',
+      default: 'sha2-256',
+      describe: 'Hash function to use. Will set Cid version to 1 if used. (experimental)'
     }
   },
 
@@ -26,16 +37,16 @@ module.exports = {
       dest,
       ipfs,
       parents,
-      recursive
+      format,
+      hash
     } = argv
 
-    ipfs.mfs.cp(source, dest, {
-      parents,
-      recursive
-    }, (error) => {
-      if (error) {
-        throw error
-      }
-    })
+    argv.resolve(
+      ipfs.files.cp(source, dest, {
+        parents,
+        format,
+        hashAlg: hash
+      })
+    )
   }
 }

src/cli/flush.js

@@ -0,0 +1,24 @@
+'use strict'
+
+const {
+  FILE_SEPARATOR
+} = require('../core/utils')
+
+module.exports = {
+  command: 'flush [path]',
+
+  describe: ' Flush a given path\'s data to disk',
+
+  builder: {},
+
+  handler (argv) {
+    let {
+      path,
+      ipfs
+    } = argv
+
+    argv.resolve(
+      ipfs.files.flush(path || FILE_SEPARATOR, {})
+    )
+  }
+}

src/cli/index.js

@@ -0,0 +1,24 @@
+'use strict'
+
+const {
+  print
+} = require('./utils')
+
+const command = {
+  command: 'files <command>',
+
+  description: 'Operations over mfs files (ls, mkdir, rm, etc)',
+
+  builder (yargs) {
+    return yargs.commandDir('.')
+  },
+
+  handler (argv) {
+    print('Type `jsipfs files --help` for more instructions')
+  }
+}
+
+module.exports = (yargs) => {
+  return yargs
+    .command(command)
+}

src/cli/ls.js

@@ -1,19 +1,24 @@
 'use strict'
 
 const {
-  print
+  print,
+  asBoolean
 } = require('./utils')
+const {
+  FILE_SEPARATOR
+} = require('../core/utils')
 
 module.exports = {
-  command: 'ls <path>',
+  command: 'ls [path]',
 
-  describe: 'List directories in the local mutable namespace.',
+  describe: 'List mfs directories',
 
   builder: {
     long: {
       alias: 'l',
       type: 'boolean',
       default: false,
+      coerce: asBoolean,
       describe: 'Use long listing format.'
     }
   },
@@ -25,20 +30,47 @@ module.exports = {
       long
     } = argv
 
-    ipfs.mfs.ls(path, {
-      long
-    }, (error, files) => {
-      if (error) {
-        throw error
-      }
-
-      files.forEach(link => {
-        if (long) {
-          return print(`${link.name} ${link.hash} ${link.size}`)
-        }
-
-        print(link.name)
-      })
-    })
+    argv.resolve(
+      ipfs.files.ls(path || FILE_SEPARATOR)
+        .then(files => {
+          if (long) {
+            const table = []
+            const lengths = {}
+
+            files.forEach(link => {
+              const row = {
+                name: `${link.name}`,
+                hash: `${link.hash}`,
+                size: `${link.size}`
+              }
+
+              Object.keys(row).forEach(key => {
+                const value = row[key]
+
+                lengths[key] = lengths[key] > value.length ? lengths[key] : value.length
+              })
+
+              table.push(row)
+            })
+
+            table.forEach(row => {
+              let line = ''
+
+              Object.keys(row).forEach(key => {
+                const value = row[key]
+
+                line += value.padEnd(lengths[key])
+                line += '\t'
+              })
+
+              print(line)
+            })
+
+            return
+          }
+
+          files.forEach(link => print(link.name))
+        })
+    )
   }
 }

src/cli/mkdir.js

@@ -1,19 +1,20 @@
 'use strict'
 
 const {
-  print
+  asBoolean
 } = require('./utils')
 
 module.exports = {
   command: 'mkdir <path>',
 
-  describe: 'Make directories.',
+  describe: 'Make mfs directories',
 
   builder: {
     parents: {
       alias: 'p',
       type: 'boolean',
       default: false,
+      coerce: asBoolean,
       describe: 'No error if existing, make parent directories as needed.'
     },
     cidVersion: {
@@ -28,6 +29,7 @@ module.exports = {
     flush: {
       alias: 'f',
       type: 'boolean',
+      coerce: asBoolean,
       describe: 'Weird undocumented option'
     }
   },
@@ -42,17 +44,13 @@ module.exports = {
       flush
     } = argv
 
-    ipfs.mfs.mkdir(path, {
-      parents,
-      cidVersion,
-      hash,
-      flush
-    }, (error, result) => {
-      if (error) {
-        throw error
-      }
-
-      print(result)
-    })
+    argv.resolve(
+      ipfs.files.mkdir(path, {
+        parents,
+        cidVersion,
+        hash,
+        flush
+      })
+    )
   }
 }

src/cli/mv.js

@@ -1,21 +1,27 @@
 'use strict'
 
+const {
+  asBoolean
+} = require('./utils')
+
 module.exports = {
   command: 'mv <source> <dest>',
 
-  describe: 'Move files around. Just like traditional unix mv',
+  describe: 'Move mfs files around',
 
   builder: {
     parents: {
       alias: 'p',
       type: 'boolean',
       default: false,
+      coerce: asBoolean,
       describe: 'Create any non-existent intermediate directories'
     },
     recursive: {
       alias: 'r',
       type: 'boolean',
       default: false,
+      coerce: asBoolean,
       describe: 'Remove directories recursively'
     }
   },
@@ -29,13 +35,11 @@ module.exports = {
       recursive
     } = argv
 
-    ipfs.mfs.mv(source, dest, {
-      parents,
-      recursive
-    }, (error) => {
-      if (error) {
-        throw error
-      }
-    })
+    argv.resolve(
+      ipfs.files.mv(source, dest, {
+        parents,
+        recursive
+      })
+    )
   }
 }