From 4ef3fa84644c80cace7934e6d06669221217653a Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 11 Dec 2018 10:00:05 +0000 Subject: [PATCH 1/9] feat: wip `addFromFs` method License: MIT Signed-off-by: Alan Shaw --- package.json | 1 + src/core/components/files-regular/add-from-fs.js | 3 +++ src/core/components/files-regular/index.js | 1 + src/core/runtime/add-from-fs-browser.js | 3 +++ src/core/runtime/add-from-fs-nodejs.js | 1 + 5 files changed, 9 insertions(+) create mode 100644 src/core/components/files-regular/add-from-fs.js create mode 100644 src/core/runtime/add-from-fs-browser.js create mode 100644 src/core/runtime/add-from-fs-nodejs.js diff --git a/package.json b/package.json index 9875580401..f022c99718 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "main": "src/core/index.js", "browser": { "./src/core/components/init-assets.js": false, + "./src/core/runtime/add-from-fs-nodejs.js": "./src/core/runtime/add-from-fs-browser.js", "./src/core/runtime/config-nodejs.js": "./src/core/runtime/config-browser.js", "./src/core/runtime/dns-nodejs.js": "./src/core/runtime/dns-browser.js", "./src/core/runtime/fetch-nodejs.js": "./src/core/runtime/fetch-browser.js", diff --git a/src/core/components/files-regular/add-from-fs.js b/src/core/components/files-regular/add-from-fs.js new file mode 100644 index 0000000000..409c4e8d7e --- /dev/null +++ b/src/core/components/files-regular/add-from-fs.js @@ -0,0 +1,3 @@ +'use strict' + +module.exports = (self) => require('../../runtime/add-from-fs-nodejs')(self) diff --git a/src/core/components/files-regular/index.js b/src/core/components/files-regular/index.js index ac3522758b..058d07d618 100644 --- a/src/core/components/files-regular/index.js +++ b/src/core/components/files-regular/index.js @@ -2,6 +2,7 @@ module.exports = self => ({ add: require('./add')(self), + addFromFs: require('./add-from-fs')(self), addFromStream: require('./add-from-stream')(self), addFromURL: require('./add-from-url')(self), addPullStream: require('./add-pull-stream')(self), diff --git a/src/core/runtime/add-from-fs-browser.js b/src/core/runtime/add-from-fs-browser.js new file mode 100644 index 0000000000..cc38fada82 --- /dev/null +++ b/src/core/runtime/add-from-fs-browser.js @@ -0,0 +1,3 @@ +'use strict' + +module.exports = () => null diff --git a/src/core/runtime/add-from-fs-nodejs.js b/src/core/runtime/add-from-fs-nodejs.js new file mode 100644 index 0000000000..ccacec309b --- /dev/null +++ b/src/core/runtime/add-from-fs-nodejs.js @@ -0,0 +1 @@ +'use strict' From a154cb5ac5ec31d162e8148607f2c15566f0f02d Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 12 Dec 2018 22:47:14 +0000 Subject: [PATCH 2/9] feat: add `addFromFs` method This PR adds a new method [`addFromFs`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#addfromfs) allowing users to more easily add files from their file system without having to specify every single file to add. In the browser the user will receive a "not available" error. I've pulled out a module `glob-source.js` - call it with some file paths and it returns a pull stream source that can be piped to `ipfs.addPullStream`. This PR comes with the following added benefits: * `ipfs add` on the CLI uses `glob-source.js` - **nice and DRY** * `glob-source.js` uses the events that the `glob` module provides allowing the globbing to be a `pull-pushable`, which means that matched paths can begin to be added before all the globbing is done - **faster** * `ipfs add` now supports adding multiple paths, fixes https://github.com/ipfs/js-ipfs/issues/1625 - **better** * `ipfs add --progress=false` doesn't calculate the total size of the files to be added anymore! It didn't need to do that as the total was completely discarded when progress was disabled. It means we can add BIGGER directories without running into memory issues - **stronger** License: MIT Signed-off-by: Alan Shaw --- package.json | 5 +- src/cli/commands/add.js | 127 ++++++------------------ src/core/runtime/add-from-fs-browser.js | 9 +- src/core/runtime/add-from-fs-nodejs.js | 19 ++++ src/utils/files/glob-source.js | 108 ++++++++++++++++++++ test/cli/files.js | 14 ++- test/core/interface.spec.js | 5 +- 7 files changed, 181 insertions(+), 106 deletions(-) create mode 100644 src/utils/files/glob-source.js diff --git a/package.json b/package.json index f022c99718..ecf20b76de 100644 --- a/package.json +++ b/package.json @@ -94,7 +94,6 @@ "datastore-core": "~0.6.0", "datastore-pubsub": "~0.1.1", "debug": "^4.1.0", - "deep-extend": "~0.6.0", "err-code": "^1.1.2", "file-type": "^10.2.0", "fnv1a": "^1.0.1", @@ -160,16 +159,14 @@ "promisify-es6": "^1.0.3", "protons": "^1.0.1", "pull-abortable": "^4.1.1", - "pull-catch": "^1.0.0", + "pull-cat": "^1.1.11", "pull-defer": "~0.2.3", "pull-file": "^1.1.0", "pull-ndjson": "~0.1.1", - "pull-paramap": "^1.2.2", "pull-pushable": "^2.2.0", "pull-sort": "^1.0.1", "pull-stream": "^3.6.9", "pull-stream-to-stream": "^1.3.4", - "pull-zip": "^2.0.1", "pump": "^3.0.0", "read-pkg-up": "^4.0.0", "readable-stream": "3.0.6", diff --git a/src/cli/commands/add.js b/src/cli/commands/add.js index 1d29e46853..dd1af23e90 100644 --- a/src/cli/commands/add.js +++ b/src/cli/commands/add.js @@ -1,86 +1,41 @@ 'use strict' -const fs = require('fs') -const path = require('path') -const glob = require('glob') const sortBy = require('lodash/sortBy') const pull = require('pull-stream') -const paramap = require('pull-paramap') -const zip = require('pull-zip') const getFolderSize = require('get-folder-size') const byteman = require('byteman') -const waterfall = require('async/waterfall') +const reduce = require('async/reduce') const mh = require('multihashes') const multibase = require('multibase') const { print, isDaemonOn, createProgressBar } = require('../utils') const { cidToString } = require('../../utils/cid') +const globSource = require('../../utils/files/glob-source') -function checkPath (inPath, recursive) { - // This function is to check for the following possible inputs - // 1) "." add the cwd but throw error for no recursion flag - // 2) "." -r return the cwd - // 3) "/some/path" but throw error for no recursion - // 4) "/some/path" -r - // 5) No path, throw err - // 6) filename.type return the cwd + filename - - if (!inPath) { - throw new Error('Error: Argument \'path\' is required') - } - - if (inPath === '.') { - inPath = process.cwd() - } - - // Convert to POSIX format - inPath = inPath - .split(path.sep) - .join('/') - - // Strips trailing slash from path. - inPath = inPath.replace(/\/$/, '') - - if (fs.statSync(inPath).isDirectory() && recursive === false) { - throw new Error(`Error: ${inPath} is a directory, use the '-r' flag to specify directories`) - } - - return inPath -} - -function getTotalBytes (path, recursive, cb) { - if (recursive) { - getFolderSize(path, cb) - } else { - fs.stat(path, (err, stat) => cb(err, stat.size)) - } +function getTotalBytes (paths, cb) { + reduce(paths, 0, (total, path, cb) => { + getFolderSize(path, (err, size) => { + if (err) return cb(err) + cb(null, total + size) + }) + }, cb) } -function addPipeline (index, addStream, list, argv) { +function addPipeline (paths, addStream, options) { const { + recursive, quiet, quieter, silent - } = argv + } = options pull( - zip( - pull.values(list), - pull( - pull.values(list), - paramap(fs.stat.bind(fs), 50) - ) - ), - pull.map((pair) => ({ - path: pair[0], - isDirectory: pair[1].isDirectory() - })), - pull.filter((file) => !file.isDirectory), - pull.map((file) => ({ - path: file.path.substring(index, file.path.length), - content: fs.createReadStream(file.path) - })), + globSource(...paths, { recursive }), addStream, pull.collect((err, added) => { if (err) { + // Tweak the error message and add more relevant infor for the CLI + if (err.code === 'ERR_DIR_NON_RECURSIVE') { + err.message = `'${err.path}' is a directory, use the '-r' flag to specify directories` + } throw err } @@ -90,10 +45,8 @@ function addPipeline (index, addStream, list, argv) { sortBy(added, 'path') .reverse() .map((file) => { - const log = [ 'added', cidToString(file.hash, { base: argv.cidBase }) ] - + const log = [ 'added', cidToString(file.hash, { base: options.cidBase }) ] if (!quiet && file.path.length > 0) log.push(file.path) - return log.join(' ') }) .forEach((msg) => print(msg)) @@ -102,7 +55,7 @@ function addPipeline (index, addStream, list, argv) { } module.exports = { - command: 'add ', + command: 'add ', describe: 'Add a file to IPFS using the UnixFS data format', @@ -191,8 +144,7 @@ module.exports = { }, handler (argv) { - const inPath = checkPath(argv.file, argv.recursive) - const index = inPath.lastIndexOf('/') + 1 + const { ipfs } = argv const options = { strategy: argv.trickle ? 'trickle' : 'balanced', shardSplitThreshold: argv.enableShardingExperiment @@ -210,38 +162,21 @@ module.exports = { if (options.enableShardingExperiment && isDaemonOn()) { throw new Error('Error: Enabling the sharding experiment should be done on the daemon') } - const ipfs = argv.ipfs - let list - waterfall([ - (next) => { - if (fs.statSync(inPath).isDirectory()) { - return glob('**/*', { cwd: inPath }, next) - } - next(null, []) - }, - (globResult, next) => { - if (globResult.length === 0) { - list = [inPath] - } else { - list = globResult.map((f) => inPath + '/' + f) - } - getTotalBytes(inPath, argv.recursive, next) - }, - (totalBytes, next) => { - if (argv.progress) { - const bar = createProgressBar(totalBytes) - options.progress = function (byteLength) { - bar.update(byteLength / totalBytes, { progress: byteman(byteLength, 2, 'MB') }) - } - } + if (!argv.progress) { + return addPipeline(argv.file, ipfs.addPullStream(options), argv) + } - next(null, ipfs.addPullStream(options)) - } - ], (err, addStream) => { + getTotalBytes(argv.file, (err, totalBytes) => { if (err) throw err - addPipeline(index, addStream, list, argv) + const bar = createProgressBar(totalBytes) + + options.progress = byteLength => { + bar.update(byteLength / totalBytes, { progress: byteman(byteLength, 2, 'MB') }) + } + + addPipeline(argv.file, ipfs.addPullStream(options), argv) }) } } diff --git a/src/core/runtime/add-from-fs-browser.js b/src/core/runtime/add-from-fs-browser.js index cc38fada82..52e56d1ea2 100644 --- a/src/core/runtime/add-from-fs-browser.js +++ b/src/core/runtime/add-from-fs-browser.js @@ -1,3 +1,10 @@ 'use strict' -module.exports = () => null +const promisify = require('promisify-es6') + +module.exports = self => { + return promisify((...args) => { + const callback = args.pop() + callback(new Error('not available in the browser')) + }) +} diff --git a/src/core/runtime/add-from-fs-nodejs.js b/src/core/runtime/add-from-fs-nodejs.js index ccacec309b..d13ba1d706 100644 --- a/src/core/runtime/add-from-fs-nodejs.js +++ b/src/core/runtime/add-from-fs-nodejs.js @@ -1 +1,20 @@ 'use strict' + +const promisify = require('promisify-es6') +const pull = require('pull-stream') +const globSource = require('../../utils/files/glob-source') +const isString = require('lodash/isString') + +module.exports = self => { + return promisify((...args) => { + const callback = args.pop() + const options = isString(args[args.length - 1]) ? {} : args.pop() + const paths = args + + pull( + globSource(...paths, options), + self.addPullStream(options), + pull.collect(callback) + ) + }) +} diff --git a/src/utils/files/glob-source.js b/src/utils/files/glob-source.js new file mode 100644 index 0000000000..9ff996b084 --- /dev/null +++ b/src/utils/files/glob-source.js @@ -0,0 +1,108 @@ +'use strict' + +const fs = require('fs') +const Path = require('path') +const isString = require('lodash/isString') +const pull = require('pull-stream') +const glob = require('glob') +const cat = require('pull-cat') +const defer = require('pull-defer') +const pushable = require('pull-pushable') +const map = require('async/map') +const parallel = require('async/parallel') +const errCode = require('err-code') + +/** +* Create a pull stream source that can be piped to ipfs.addPullStream for the +* provided file paths. +* +* @param ...paths {String} File system path(s) to glob from +* @param [options] {Object} Optional options +* @param [options.recursive] Recursively glob all paths in directories +*/ +module.exports = (...args) => { + const options = isString(args[args.length - 1]) ? {} : args.pop() + const paths = args + const deferred = defer.source() + + const globSourceOptions = { + recursive: options.recursive, + glob: { + dot: Boolean(options.hidden), + ignore: Array.isArray(options.ignore) ? options.ignore : [], + follow: options.followSymlinks != null ? options.followSymlinks : true + } + } + + // Check the input paths comply with options.recursive and convert to glob sources + map(paths, normalizePathWithType, (err, results) => { + if (err) return deferred.abort(err) + + try { + const sources = results.map(res => toGlobSource(res, globSourceOptions)) + return deferred.resolve(cat(sources)) + } catch (err) { + return deferred.abort(err) + } + }) + + return pull( + deferred, + pull.map(({ path, contentPath }) => ({ + path, + content: fs.createReadStream(contentPath) + })) + ) +} + +function toGlobSource ({ path, type }, options) { + options = options || {} + + const baseName = Path.basename(path) + + if (type === 'file') { + return pull.values([{ path: baseName, contentPath: path }]) + } + + if (type === 'dir' && !options.recursive) { + throw errCode( + new Error(`'${path}' is a directory and recursive option not set`), + 'ERR_DIR_NON_RECURSIVE', + { path } + ) + } + + const globOptions = Object.assign({}, options.glob, { + cwd: path, + nodir: true, + realpath: false, + absolute: false + }) + + // TODO: want to use pull-glob but it doesn't have the features... + const pusher = pushable() + + glob('**/*', globOptions) + .on('match', m => pusher.push(m)) + .on('end', () => pusher.end()) + .on('abort', () => pusher.end()) + .on('error', err => pusher.end(err)) + + return pull( + pusher, + pull.map(p => ({ + path: Path.join(baseName, p), + contentPath: Path.join(path, p) + })) + ) +} + +function normalizePathWithType (path, cb) { + parallel({ + stat: cb => fs.stat(path, cb), + realpath: cb => fs.realpath(path, cb) + }, (err, res) => { + if (err) return cb(err) + cb(null, { path: res.realpath, type: res.stat.isDirectory() ? 'dir' : 'file' }) + }) +} diff --git a/test/cli/files.js b/test/cli/files.js index 71f0a8a886..d9990ebd29 100644 --- a/test/cli/files.js +++ b/test/cli/files.js @@ -143,6 +143,18 @@ describe('files', () => runOnAndOff((thing) => { }) }) + it('add multiple', function () { + this.timeout(30 * 1000) + + return ipfs('add', 'src/init-files/init-docs/readme', 'test/fixtures/odd-name-[v0]/odd name [v1]/hello', '--wrap-with-directory') + .then((out) => { + expect(out) + .to.include('added QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB readme\n') + expect(out) + .to.include('added QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o hello\n') + }) + }) + it('add alias', function () { this.timeout(30 * 1000) @@ -278,7 +290,7 @@ describe('files', () => runOnAndOff((thing) => { it('add --quieter', function () { this.timeout(30 * 1000) - return ipfs('add -Q -w test/fixtures/test-data/hello test/test-data/node.json') + return ipfs('add -Q -w test/fixtures/test-data/hello') .then((out) => { expect(out) .to.eql('QmYRMUVULBfj7WrdPESnwnyZmtayN6Sdrwh1nKcQ9QgQeZ\n') diff --git a/test/core/interface.spec.js b/test/core/interface.spec.js index 6a31a90958..05284161b7 100644 --- a/test/core/interface.spec.js +++ b/test/core/interface.spec.js @@ -51,10 +51,7 @@ describe('interface-ipfs-core tests', () => { }) tests.filesRegular(defaultCommonFactory, { - skip: isNode ? [{ - name: 'addFromFs', - reason: 'TODO: not implemented yet' - }] : [{ + skip: isNode ? null : [{ name: 'addFromStream', reason: 'Not designed to run in the browser' }, { From 59a3fab951c4de98ffcb9fe9defbd754651b271a Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 13 Dec 2018 10:38:06 +0000 Subject: [PATCH 3/9] docs: added jsdocs for options for glob-source.js License: MIT Signed-off-by: Alan Shaw --- src/utils/files/glob-source.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/utils/files/glob-source.js b/src/utils/files/glob-source.js index 9ff996b084..517c11c83f 100644 --- a/src/utils/files/glob-source.js +++ b/src/utils/files/glob-source.js @@ -18,7 +18,10 @@ const errCode = require('err-code') * * @param ...paths {String} File system path(s) to glob from * @param [options] {Object} Optional options -* @param [options.recursive] Recursively glob all paths in directories +* @param [options.recursive] {Boolean} Recursively glob all paths in directories +* @param [options.hidden] {Boolean} Include .dot files in matched paths +* @param [options.ignore] {Array} Glob paths to ignore +* @param [options.followSymlinks] {Boolean} follow symlinks */ module.exports = (...args) => { const options = isString(args[args.length - 1]) ? {} : args.pop() From 976497bcbef6b6b1c55b8704ec0e70d82e1efb45 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 13 Dec 2018 11:25:57 +0000 Subject: [PATCH 4/9] chore: appease linter License: MIT Signed-off-by: Alan Shaw --- src/utils/files/glob-source.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/utils/files/glob-source.js b/src/utils/files/glob-source.js index 517c11c83f..5df2eb8e1a 100644 --- a/src/utils/files/glob-source.js +++ b/src/utils/files/glob-source.js @@ -16,12 +16,13 @@ const errCode = require('err-code') * Create a pull stream source that can be piped to ipfs.addPullStream for the * provided file paths. * -* @param ...paths {String} File system path(s) to glob from -* @param [options] {Object} Optional options -* @param [options.recursive] {Boolean} Recursively glob all paths in directories -* @param [options.hidden] {Boolean} Include .dot files in matched paths -* @param [options.ignore] {Array} Glob paths to ignore -* @param [options.followSymlinks] {Boolean} follow symlinks +* @param {String} ...paths File system path(s) to glob from +* @param {Object} [options] Optional options +* @param {Boolean} [options.recursive] Recursively glob all paths in directories +* @param {Boolean} [options.hidden] Include .dot files in matched paths +* @param {Array} [options.ignore] Glob paths to ignore +* @param {Boolean} [options.followSymlinks] follow symlinks +* @returns {Function} pull stream source */ module.exports = (...args) => { const options = isString(args[args.length - 1]) ? {} : args.pop() From db3c18353ebc80869161f2c37fea602b9ba26d91 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Sat, 15 Dec 2018 19:52:37 +0000 Subject: [PATCH 5/9] refactor: use typeof instead of lodash/isString License: MIT Signed-off-by: Alan Shaw --- src/core/runtime/add-from-fs-nodejs.js | 3 +-- src/utils/files/glob-source.js | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/core/runtime/add-from-fs-nodejs.js b/src/core/runtime/add-from-fs-nodejs.js index d13ba1d706..cff5375fbd 100644 --- a/src/core/runtime/add-from-fs-nodejs.js +++ b/src/core/runtime/add-from-fs-nodejs.js @@ -3,12 +3,11 @@ const promisify = require('promisify-es6') const pull = require('pull-stream') const globSource = require('../../utils/files/glob-source') -const isString = require('lodash/isString') module.exports = self => { return promisify((...args) => { const callback = args.pop() - const options = isString(args[args.length - 1]) ? {} : args.pop() + const options = typeof args[args.length - 1] === 'string' ? {} : args.pop() const paths = args pull( diff --git a/src/utils/files/glob-source.js b/src/utils/files/glob-source.js index 5df2eb8e1a..36747f85d6 100644 --- a/src/utils/files/glob-source.js +++ b/src/utils/files/glob-source.js @@ -2,7 +2,6 @@ const fs = require('fs') const Path = require('path') -const isString = require('lodash/isString') const pull = require('pull-stream') const glob = require('glob') const cat = require('pull-cat') @@ -25,7 +24,7 @@ const errCode = require('err-code') * @returns {Function} pull stream source */ module.exports = (...args) => { - const options = isString(args[args.length - 1]) ? {} : args.pop() + const options = typeof args[args.length - 1] === 'string' ? {} : args.pop() const paths = args const deferred = defer.source() From 384a9e6bb0d9a8f5a4dfe6d4e17a62830a6e543d Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Sun, 16 Dec 2018 21:31:47 +0000 Subject: [PATCH 6/9] fix: path is always posix License: MIT Signed-off-by: Alan Shaw --- src/utils/files/glob-source.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/utils/files/glob-source.js b/src/utils/files/glob-source.js index 36747f85d6..be3c2d59d9 100644 --- a/src/utils/files/glob-source.js +++ b/src/utils/files/glob-source.js @@ -106,6 +106,9 @@ function normalizePathWithType (path, cb) { realpath: cb => fs.realpath(path, cb) }, (err, res) => { if (err) return cb(err) - cb(null, { path: res.realpath, type: res.stat.isDirectory() ? 'dir' : 'file' }) + cb(null, { + path: res.realpath.replace(/\\/g, '/'), + type: res.stat.isDirectory() ? 'dir' : 'file' + }) }) } From 158d37d459799c464b5c3fb03af5d3e8e908b1d8 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 17 Dec 2018 09:28:45 +0000 Subject: [PATCH 7/9] fix: fixes for windows License: MIT Signed-off-by: Alan Shaw --- src/utils/files/glob-source.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/utils/files/glob-source.js b/src/utils/files/glob-source.js index be3c2d59d9..5f79f8e933 100644 --- a/src/utils/files/glob-source.js +++ b/src/utils/files/glob-source.js @@ -43,9 +43,9 @@ module.exports = (...args) => { try { const sources = results.map(res => toGlobSource(res, globSourceOptions)) - return deferred.resolve(cat(sources)) + deferred.resolve(cat(sources)) } catch (err) { - return deferred.abort(err) + deferred.abort(err) } }) @@ -94,7 +94,7 @@ function toGlobSource ({ path, type }, options) { return pull( pusher, pull.map(p => ({ - path: Path.join(baseName, p), + path: `${baseName}/${toPosix(p)}`, contentPath: Path.join(path, p) })) ) @@ -107,8 +107,10 @@ function normalizePathWithType (path, cb) { }, (err, res) => { if (err) return cb(err) cb(null, { - path: res.realpath.replace(/\\/g, '/'), + path: toPosix(res.realpath), type: res.stat.isDirectory() ? 'dir' : 'file' }) }) } + +const toPosix = path => path.replace(/\\/g, '/') From 4258046dadc357e87dd3ba5db445c269dde09c33 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 17 Dec 2018 12:33:56 +0000 Subject: [PATCH 8/9] fix: remove unnecessary realpath call License: MIT Signed-off-by: Alan Shaw --- src/utils/files/glob-source.js | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/utils/files/glob-source.js b/src/utils/files/glob-source.js index 5f79f8e933..8c6368f043 100644 --- a/src/utils/files/glob-source.js +++ b/src/utils/files/glob-source.js @@ -8,7 +8,6 @@ const cat = require('pull-cat') const defer = require('pull-defer') const pushable = require('pull-pushable') const map = require('async/map') -const parallel = require('async/parallel') const errCode = require('err-code') /** @@ -38,7 +37,7 @@ module.exports = (...args) => { } // Check the input paths comply with options.recursive and convert to glob sources - map(paths, normalizePathWithType, (err, results) => { + map(paths, pathAndType, (err, results) => { if (err) return deferred.abort(err) try { @@ -100,16 +99,10 @@ function toGlobSource ({ path, type }, options) { ) } -function normalizePathWithType (path, cb) { - parallel({ - stat: cb => fs.stat(path, cb), - realpath: cb => fs.realpath(path, cb) - }, (err, res) => { +function pathAndType (path, cb) { + fs.stat(path, (err, stat) => { if (err) return cb(err) - cb(null, { - path: toPosix(res.realpath), - type: res.stat.isDirectory() ? 'dir' : 'file' - }) + cb(null, { path, type: stat.isDirectory() ? 'dir' : 'file' }) }) } From 43fc57317e17bb52b3b25520d9d327e1197e9118 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 17 Dec 2018 16:16:06 +0000 Subject: [PATCH 9/9] fix: do not rely on external has being available License: MIT Signed-off-by: Alan Shaw --- test/http-api/inject/files.js | 36 ++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/test/http-api/inject/files.js b/test/http-api/inject/files.js index 66c4e8a1f6..677c4c43fa 100644 --- a/test/http-api/inject/files.js +++ b/test/http-api/inject/files.js @@ -106,16 +106,34 @@ module.exports = (http) => { }) }) - it('valid hash', function (done) { + it('should cat a valid hash', function (done) { this.timeout(30 * 1000) - api.inject({ - method: 'GET', - url: '/api/v0/cat?arg=QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o' - }, (res) => { - expect(res.statusCode).to.equal(200) - expect(res.rawPayload).to.deep.equal(Buffer.from('hello world' + '\n')) - expect(res.payload).to.equal('hello world' + '\n') - done() + + const data = Buffer.from('TEST' + Date.now()) + const form = new FormData() + form.append('data', data) + const headers = form.getHeaders() + + streamToPromise(form).then((payload) => { + api.inject({ + method: 'POST', + url: '/api/v0/add', + headers: headers, + payload: payload + }, (res) => { + expect(res.statusCode).to.equal(200) + const cid = JSON.parse(res.result).Hash + + api.inject({ + method: 'GET', + url: '/api/v0/cat?arg=' + cid + }, (res) => { + expect(res.statusCode).to.equal(200) + expect(res.rawPayload).to.deep.equal(data) + expect(res.payload).to.equal(data.toString()) + done() + }) + }) }) }) })