feat: support cidv1b32 in resolver

- feat: CID support, added resolver.cid
- feat: basic support for HAMD sharded directory
  - not real support, we need ipfs.resolve for that
- fix: return data from raw dag without resolv step

Known Issues:
- missing tests for CIDv1

License: MIT
Signed-off-by: Marcin Rataj <[email protected]>

Author: lidel

README.md

@@ -18,7 +18,10 @@
 
 ## Usage
 
-This project consists on creating a HTTP response from an IPFS Hash. This response can be a file, a directory list view or the entry point of a web page.
+
+### Creating HTTP Response
+
+This project creates a HTTP response for an IPFS Path. This response can be a file, a HTML with directory listing or the entry point of a web page.
 
 ```js
 const { getResponse } = require('ipfs-http-response')
@@ -29,24 +32,31 @@ getResponse(ipfsNode, ipfsPath)
   })
 ```
 
-This module also exports the used ipfs resolver, which should be used when the response needs to be customized.
+### Using protocol-agnostic resolver
+
+This module also exports the used ipfs `resolver`, which should be used when the response needs to be customized or non-HTTP transport is used:
 
 ```js
 const { resolver } = require('ipfs-http-response')
 
-resolver.multihash(ipfsNode, ipfsPath)
+resolver.cid(ipfsNode, ipfsPath)
   .then((result) => {
     ...
   })
 ```
 
+If `ipfsPath` points at a directory, `resolver.cid` will throw Error `This dag node is a directory` with a `cid` attribute that can be passed to `resolver.directory`:
+
+
 ```js
 const { resolver } = require('ipfs-http-response')
 
-resolver.directory(node, path, multihash)
+resolver.directory(ipfsNode, ipfsPath, cid)
   .then((result) => {
     ...
   })
 ```
 
+`result` will be either a `string` with HTML directory listing or an array with CIDs of `index` pages present in inspected directory.
+
 ![ipfs-http-response usage](docs/ipfs-http-response.png "ipfs-http-response usage")

package.json

@@ -46,8 +46,8 @@
     "aegir": "^13.1.0",
     "chai": "^4.1.2",
     "dirty-chai": "^2.0.1",
-    "ipfs": "^0.28.2",
-    "ipfsd-ctl": "^0.36.0"
+    "ipfs": "^0.31.4",
+    "ipfsd-ctl": "^0.39.1"
   },
   "contributors": [
     "André Cruz <[email protected]>",

src/dir-view/index.js

@@ -5,21 +5,20 @@ const filesize = require('filesize')
 const mainStyle = require('./style')
 const pathUtil = require('../utils/path')
 
-function getParentDirectoryURL (originalParts) {
-  const parts = originalParts.slice()
-
+function getParentHref (path) {
+  const parts = pathUtil.cidArray(path).slice()
   if (parts.length > 1) {
-    parts.pop()
+    // drop the last segment in a safe way that works for both paths and urls
+    return path.replace(`/${parts.pop()}`, '')
   }
-
-  return [ '', 'ipfs' ].concat(parts).join('/')
+  return path
 }
 
 function buildFilesList (path, links) {
   const rows = links.map((link) => {
     let row = [
       `<div class="ipfs-icon ipfs-_blank">&nbsp;</div>`,
-      `<a href="${pathUtil.joinURLParts(path, link.name)}">${link.name}</a>`,
+      `<a href="${path}${path.endsWith('/') ? '' : '/'}${link.name}">${link.name}</a>`,
       filesize(link.size)
     ]
 
@@ -32,9 +31,6 @@ function buildFilesList (path, links) {
 }
 
 function buildTable (path, links) {
-  const parts = pathUtil.splitPath(path)
-  const parentDirectoryURL = getParentDirectoryURL(parts)
-
   return `
     <table class="table table-striped">
       <tbody>
@@ -43,7 +39,7 @@ function buildTable (path, links) {
             <div class="ipfs-icon ipfs-_blank">&nbsp;</div>
           </td>
           <td class="padding">
-            <a href="${parentDirectoryURL}">..</a>
+            <a href="${getParentHref(path)}">..</a>
           </td>
           <td></td>
         </tr>

src/index.js

@@ -25,7 +25,7 @@ const response = (ipfsNode, ipfsPath) => {
         // switch case with true feels so wrong.
         switch (true) {
           case (errorString === 'Error: This dag node is a directory'):
-            resolver.directory(node, path, error.fileName)
+            resolver.directory(node, path, error.cid)
               .then((content) => {
                 // dir render
                 if (typeof content === 'string') {
@@ -59,9 +59,9 @@ const response = (ipfsNode, ipfsPath) => {
       resolve(Response.redirect(pathUtils.removeTrailingSlash(ipfsPath)))
     }
 
-    resolver.multihash(ipfsNode, ipfsPath)
+    resolver.cid(ipfsNode, ipfsPath)
       .then((resolvedData) => {
-        const readableStream = ipfsNode.files.catReadableStream(resolvedData.multihash)
+        const readableStream = ipfsNode.files.catReadableStream(resolvedData.cid)
         const responseStream = new stream.PassThrough({ highWaterMark: 1 })
         readableStream.pipe(responseStream)
 

src/resolver.js

@@ -18,14 +18,21 @@ function getIndexFiles (links) {
     'index.htm',
     'index.shtml'
   ]
-
-  return links.filter((link) => INDEX_HTML_FILES.indexOf(link.name) !== -1)
+  // directory
+  let indexes = links.filter((link) => INDEX_HTML_FILES.indexOf(link.name) !== -1)
+  if (indexes.length) {
+    return indexes
+  }
+  // hamt-sharded-directory uses a 2 char prefix
+  return links.filter((link) => {
+    return link.name.length > 2 && INDEX_HTML_FILES.indexOf(link.name.substring(2)) !== -1
+  })
 }
 
-const directory = promisify((ipfs, path, multihash, callback) => {
-  mh.validate(mh.fromB58String(multihash))
+const directory = promisify((ipfs, path, cid, callback) => {
+  cid = new CID(cid)
 
-  ipfs.object.get(multihash, { enc: 'base58' }, (err, dagNode) => {
+  ipfs.object.get(cid.buffer, (err, dagNode) => {
     if (err) {
       return callback(err)
     }
@@ -41,17 +48,20 @@ const directory = promisify((ipfs, path, multihash, callback) => {
   })
 })
 
-const multihash = promisify((ipfs, path, callback) => {
-  const parts = pathUtil.splitPath(path)
-  let firstMultihash = parts.shift()
+const cid = promisify((ipfs, path, callback) => {
+  const parts = pathUtil.cidArray(path)
+  let firstCid = parts.shift()
   let currentCid
 
+  // TODO: replace below with ipfs.resolve(path, {recursive: true})
+  // (requires changes to js-ipfs/js-ipfs-api)
+
   reduce(
     parts,
-    firstMultihash,
+    firstCid,
     (memo, item, next) => {
       try {
-        currentCid = new CID(mh.fromB58String(memo))
+        currentCid = new CID(memo)
       } catch (err) {
         return next(err)
       }
@@ -65,56 +75,90 @@ const multihash = promisify((ipfs, path, callback) => {
         }
 
         const dagNode = result.value
-        // find multihash of requested named-file in current dagNode's links
-        let multihashOfNextFile
+        // find multihash/cid of requested named-file in current dagNode's links
+        let cidOfNextFile
         const nextFileName = item
 
-        for (let link of dagNode.links) {
-          if (link.name === nextFileName) {
-            // found multihash of requested named-file
-            multihashOfNextFile = mh.toB58String(link.multihash)
-            log('found multihash: ', multihashOfNextFile)
-            break
+        try {
+          for (let link of dagNode.links) {
+            if (link.name === nextFileName) {
+              // found multihash/cid of requested named-file
+              try {
+                // assume a Buffer with a valid CID
+                // (cid is allowed instead of multihash since https://github.com/ipld/js-ipld-dag-pb/pull/80)
+                cidOfNextFile = new CID(link.multihash)
+              } catch (err) {
+                // fallback to multihash
+                cidOfNextFile = new CID(mh.toB58String(link.multihash))
+              }
+              break
+            }
           }
+        } catch (err) {
+          return next(err)
         }
 
-        if (!multihashOfNextFile) {
-          return next(new Error(`no link named "${nextFileName}" under ${memo}`))
+        if (!cidOfNextFile) {
+          const missingLinkErr = new Error(`no link named "${nextFileName}" under ${memo}`)
+          missingLinkErr.parentDagNode = memo
+          missingLinkErr.missingLinkName = nextFileName
+          return next(missingLinkErr)
         }
 
-        next(null, multihashOfNextFile)
+        next(null, cidOfNextFile)
       })
-    }, (err, result) => {
+    }, (err, cid) => {
       if (err) {
         return callback(err)
       }
 
-      let cid
       try {
-        cid = new CID(mh.fromB58String(result))
+        cid = new CID(cid)
       } catch (err) {
         return callback(err)
       }
 
+      if (cid.codec === 'raw') {
+        // no need for additional lookup, its raw data
+        callback(null, { cid })
+      }
+
       ipfs.dag.get(cid, (err, dagResult) => {
         if (err) {
           return callback(err)
         }
 
-        let dagDataObj = Unixfs.unmarshal(dagResult.value.data)
-        if (dagDataObj.type === 'directory') {
-          let isDirErr = new Error('This dag node is a directory')
-          // add memo (last multihash) as a fileName so it can be used by directory
-          isDirErr.fileName = result
-          return callback(isDirErr)
+        try {
+          let dagDataObj = Unixfs.unmarshal(dagResult.value.data)
+          // There are at least two types of directories:
+          // - "directory"
+          // - "hamt-sharded-directory" (example: QmT5NvUtoM5nWFfrQdVrFtvGfKFmG7AHE8P34isapyhCxX)
+          if (dagDataObj.type === 'directory' || dagDataObj.type === 'hamt-sharded-directory') {
+            let isDirErr = new Error('This dag node is a directory')
+            // store memo of last multihash so it can be used by directory
+            isDirErr.cid = cid
+            isDirErr.dagDirType = dagDataObj.type
+            return callback(isDirErr)
+          }
+        } catch (err) {
+          return callback(err)
         }
 
-        callback(null, { multihash: result })
+        callback(null, { cid })
       })
     })
 })
 
+const multihash = promisify((ipfs, path, callback) => {
+  // deprecated, use 'cid' instead
+  // (left for backward-compatibility)
+  cid(ipfs, path)
+    .then((result) => { callback(null, { multihash: mh.toB58String(result.cid.multihash) }) })
+    .catch((err) => { callback(err) })
+})
+
 module.exports = {
   directory: directory,
+  cid: cid,
   multihash: multihash
 }

src/utils/path.js

@@ -1,12 +1,21 @@
 'use strict'
 
 /* eslint-disable no-unused-vars */
-function splitPath (path) {
+
+// Converts path or url to an array starting at CID
+function cidArray (path) {
   if (path[path.length - 1] === '/') {
     path = path.substring(0, path.length - 1)
   }
-
-  return path.substring(6).split('/')
+  // skip /ipxs/ prefix
+  if (path.match(/^\/ip[fn]s\//)) {
+    path = path.substring(6)
+  }
+  // skip ipxs:// protocol
+  if (path.match(/^ip[fn]s:\/\//)) {
+    path = path.substring(7)
+  }
+  return path.split('/')
 }
 
 function removeLeadingSlash (url) {
@@ -40,7 +49,7 @@ function joinURLParts (...urls) {
 }
 
 module.exports = {
-  splitPath,
+  cidArray,
   removeLeadingSlash,
   removeTrailingSlash,
   removeSlashFromBothEnds,