Merge pull request #5789 from ipfs/kevina/multibase4

Add global option to specify the multibase encoding (server side)

Author: Stebalien

core/commands/add.go

@@ -174,6 +174,11 @@ You can now check what blocks have been created by:
 			return fmt.Errorf("unrecognized hash function: %s", strings.ToLower(hashFunStr))
 		}
 
+		enc, err := cmdenv.GetCidEncoder(req)
+		if err != nil {
+			return err
+		}
+
 		events := make(chan interface{}, adderOutChanSize)
 
 		opts := []options.UnixfsAddOption{
@@ -226,7 +231,7 @@ You can now check what blocks have been created by:
 
 			h := ""
 			if output.Path != nil {
-				h = output.Path.Cid().String()
+				h = enc.Encode(output.Path.Cid())
 			}
 
 			res.Emit(&AddEvent{

core/commands/bitswap.go

@@ -12,7 +12,7 @@ import (
 	peer "gx/ipfs/QmY5Grm8pJdiSSVsYxx4uNRgweY72EmYwuSDbRnbFok3iY/go-libp2p-peer"
 	bitswap "gx/ipfs/QmYoGLuLwTUv1SYBmsw1EVNC9MyLVUxwxzXYtKgAGHyEfw/go-bitswap"
 	decision "gx/ipfs/QmYoGLuLwTUv1SYBmsw1EVNC9MyLVUxwxzXYtKgAGHyEfw/go-bitswap/decision"
-	cidutil "gx/ipfs/QmbfKu17LbMWyGUxHEUns9Wf5Dkm8PT6be4uPhTkk4YvaV/go-cidutil"
+	cidutil "gx/ipfs/QmdPQx9fvN5ExVwMhRmh7YpCQJzJrFhd1AjVBwJmRMFJeX/go-cidutil"
 	cmdkit "gx/ipfs/Qmde5VP1qUkyQXKCfmEUA7bP64V2HAptbJ7phuPp7jXWwg/go-ipfs-cmdkit"
 )
 
@@ -74,12 +74,15 @@ Print out all blocks currently on the bitswap wantlist for the local peer.`,
 	},
 	Encoders: cmds.EncoderMap{
 		cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *KeyList) error {
+			enc, err := cmdenv.GetLowLevelCidEncoder(req)
+			if err != nil {
+				return err
+			}
 			// sort the keys first
 			cidutil.Sort(out.Keys)
 			for _, key := range out.Keys {
-				fmt.Fprintln(w, key)
+				fmt.Fprintln(w, enc.Encode(key))
 			}
-
 			return nil
 		}),
 	},
@@ -115,6 +118,10 @@ var bitswapStatCmd = &cmds.Command{
 	},
 	Encoders: cmds.EncoderMap{
 		cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, s *bitswap.Stat) error {
+			enc, err := cmdenv.GetLowLevelCidEncoder(req)
+			if err != nil {
+				return err
+			}
 			fmt.Fprintln(w, "bitswap status")
 			fmt.Fprintf(w, "\tprovides buffer: %d / %d\n", s.ProvideBufLen, bitswap.HasBlockBufferSize)
 			fmt.Fprintf(w, "\tblocks received: %d\n", s.BlocksReceived)
@@ -125,7 +132,7 @@ var bitswapStatCmd = &cmds.Command{
 			fmt.Fprintf(w, "\tdup data received: %s\n", humanize.Bytes(s.DupDataReceived))
 			fmt.Fprintf(w, "\twantlist [%d keys]\n", len(s.Wantlist))
 			for _, k := range s.Wantlist {
-				fmt.Fprintf(w, "\t\t%s\n", k.String())
+				fmt.Fprintf(w, "\t\t%s\n", enc.Encode(k))
 			}
 			fmt.Fprintf(w, "\tpartners [%d]\n", len(s.Peers))
 			for _, p := range s.Peers {

core/commands/cid.go

@@ -10,7 +10,7 @@ import (
 	cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid"
 	cmds "gx/ipfs/QmWGm4AbZEbnmdgVTza52MSNpEmBdFVqzmAysRbjrRyGbH/go-ipfs-cmds"
 	verifcid "gx/ipfs/QmYMQuypUbgsdNHmuCBSUJV6wdQVsBHRivNAp3efHJwZJD/go-verifcid"
-	cidutil "gx/ipfs/QmbfKu17LbMWyGUxHEUns9Wf5Dkm8PT6be4uPhTkk4YvaV/go-cidutil"
+	cidutil "gx/ipfs/QmdPQx9fvN5ExVwMhRmh7YpCQJzJrFhd1AjVBwJmRMFJeX/go-cidutil"
 	cmdkit "gx/ipfs/Qmde5VP1qUkyQXKCfmEUA7bP64V2HAptbJ7phuPp7jXWwg/go-ipfs-cmdkit"
 	mbase "gx/ipfs/QmekxXDhCxCJRNuzmHreuaT3BsuJcsjcXWNrtV9C8DRHtd/go-multibase"
 	mhash "gx/ipfs/QmerPMzPk1mJVowm8KgmoknWa4yCYvvugMPsgWmDNUvDLW/go-multihash"

core/commands/cmdenv/cidbase.go

@@ -0,0 +1,107 @@
+package cmdenv
+
+import (
+	"fmt"
+	"strings"
+
+	cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid"
+	cmds "gx/ipfs/QmWGm4AbZEbnmdgVTza52MSNpEmBdFVqzmAysRbjrRyGbH/go-ipfs-cmds"
+	cidenc "gx/ipfs/QmdPQx9fvN5ExVwMhRmh7YpCQJzJrFhd1AjVBwJmRMFJeX/go-cidutil/cidenc"
+	cmdkit "gx/ipfs/Qmde5VP1qUkyQXKCfmEUA7bP64V2HAptbJ7phuPp7jXWwg/go-ipfs-cmdkit"
+	mbase "gx/ipfs/QmekxXDhCxCJRNuzmHreuaT3BsuJcsjcXWNrtV9C8DRHtd/go-multibase"
+)
+
+var OptionCidBase = cmdkit.StringOption("cid-base", "Multibase encoding used for version 1 CIDs in output.")
+var OptionUpgradeCidV0InOutput = cmdkit.BoolOption("upgrade-cidv0-in-output", "Upgrade version 0 to version 1 CIDs in output.")
+
+// GetCidEncoder processes the `cid-base` and `output-cidv1` options and
+// returns a encoder to use based on those parameters.
+func GetCidEncoder(req *cmds.Request) (cidenc.Encoder, error) {
+	return getCidBase(req, true)
+}
+
+// GetLowLevelCidEncoder is like GetCidEncoder but meant to be used by
+// lower level commands.  It differs from GetCidEncoder in that CIDv0
+// are not, by default, auto-upgraded to CIDv1.
+func GetLowLevelCidEncoder(req *cmds.Request) (cidenc.Encoder, error) {
+	return getCidBase(req, false)
+}
+
+func getCidBase(req *cmds.Request, autoUpgrade bool) (cidenc.Encoder, error) {
+	base, _ := req.Options[OptionCidBase.Name()].(string)
+	upgrade, upgradeDefined := req.Options[OptionUpgradeCidV0InOutput.Name()].(bool)
+
+	e := cidenc.Default()
+
+	if base != "" {
+		var err error
+		e.Base, err = mbase.EncoderByName(base)
+		if err != nil {
+			return e, err
+		}
+		if autoUpgrade {
+			e.Upgrade = true
+		}
+	}
+
+	if upgradeDefined {
+		e.Upgrade = upgrade
+	}
+
+	return e, nil
+}
+
+// CidBaseDefined returns true if the `cid-base` option is specified
+// on the command line
+func CidBaseDefined(req *cmds.Request) bool {
+	base, _ := req.Options["cid-base"].(string)
+	return base != ""
+}
+
+// CidEncoderFromPath creates a new encoder that is influenced from
+// the encoded Cid in a Path.  For CidV0 the multibase from the base
+// encoder is used and automatic upgrades are disabled.  For CidV1 the
+// multibase from the CID is used and upgrades are enabled.
+//
+// This logic is intentionally fuzzy and will match anything of the form
+// `CidLike`, `CidLike/...`, or `/namespace/CidLike/...`.
+//
+// For example:
+//
+// * Qm...
+// * Qm.../...
+// * /ipfs/Qm...
+// * /ipns/bafybeiahnxfi7fpmr5wtxs2imx4abnyn7fdxeiox7xxjem6zuiioqkh6zi/...
+// * /bzz/bafybeiahnxfi7fpmr5wtxs2imx4abnyn7fdxeiox7xxjem6zuiioqkh6zi/...
+func CidEncoderFromPath(p string) (cidenc.Encoder, error) {
+	components := strings.SplitN(p, "/", 4)
+
+	var maybeCid string
+	if components[0] != "" {
+		// No leading slash, first component is likely CID-like.
+		maybeCid = components[0]
+	} else if len(components) < 3 {
+		// Not enough components to include a CID.
+		return cidenc.Encoder{}, fmt.Errorf("no cid in path: %s", p)
+	} else {
+		maybeCid = components[2]
+	}
+	c, err := cid.Decode(maybeCid)
+	if err != nil {
+		// Ok, not a CID-like thing. Keep the current encoder.
+		return cidenc.Encoder{}, fmt.Errorf("no cid in path: %s", p)
+	}
+	if c.Version() == 0 {
+		// Version 0, use the base58 non-upgrading encoder.
+		return cidenc.Default(), nil
+	}
+
+	// Version 1+, extract multibase encoding.
+	encoding, _, err := mbase.Decode(maybeCid)
+	if err != nil {
+		// This should be impossible, we've already decoded the cid.
+		panic(fmt.Sprintf("BUG: failed to get multibase decoder for CID %s", maybeCid))
+	}
+
+	return cidenc.Encoder{Base: mbase.MustNewEncoder(encoding), Upgrade: true}, nil
+}

core/commands/cmdenv/cidbase_test.go

@@ -0,0 +1,73 @@
+package cmdenv
+
+import (
+	"testing"
+
+	cidenc "gx/ipfs/QmdPQx9fvN5ExVwMhRmh7YpCQJzJrFhd1AjVBwJmRMFJeX/go-cidutil/cidenc"
+	mbase "gx/ipfs/QmekxXDhCxCJRNuzmHreuaT3BsuJcsjcXWNrtV9C8DRHtd/go-multibase"
+)
+
+func TestEncoderFromPath(t *testing.T) {
+	test := func(path string, expected cidenc.Encoder) {
+		actual, err := CidEncoderFromPath(path)
+		if err != nil {
+			t.Error(err)
+		}
+		if actual != expected {
+			t.Errorf("CidEncoderFromPath(%s) failed: expected %#v but got %#v", path, expected, actual)
+		}
+	}
+	p := "QmRqVG8VGdKZ7KARqR96MV7VNHgWvEQifk94br5HpURpfu"
+	enc := cidenc.Default()
+	test(p, enc)
+	test(p+"/a", enc)
+	test(p+"/a/b", enc)
+	test(p+"/a/b/", enc)
+	test(p+"/a/b/c", enc)
+	test("/ipfs/"+p, enc)
+	test("/ipfs/"+p+"/b", enc)
+
+	p = "zb2rhfkM4FjkMLaUnygwhuqkETzbYXnUDf1P9MSmdNjW1w1Lk"
+	enc = cidenc.Encoder{
+		Base:    mbase.MustNewEncoder(mbase.Base58BTC),
+		Upgrade: true,
+	}
+	test(p, enc)
+	test(p+"/a", enc)
+	test(p+"/a/b", enc)
+	test(p+"/a/b/", enc)
+	test(p+"/a/b/c", enc)
+	test("/ipfs/"+p, enc)
+	test("/ipfs/"+p+"/b", enc)
+	test("/ipld/"+p, enc)
+	test("/ipns/"+p, enc) // even IPNS should work.
+
+	p = "bafyreifrcnyjokuw4i4ggkzg534tjlc25lqgt3ttznflmyv5fftdgu52hm"
+	enc = cidenc.Encoder{
+		Base:    mbase.MustNewEncoder(mbase.Base32),
+		Upgrade: true,
+	}
+	test(p, enc)
+	test("/ipfs/"+p, enc)
+	test("/ipld/"+p, enc)
+
+	for _, badPath := range []string{
+		"/ipld/",
+		"/ipld",
+		"/ipld//",
+		"ipld//",
+		"ipld",
+		"",
+		"ipns",
+		"/ipfs/asdf",
+		"/ipfs/...",
+		"...",
+		"abcdefg",
+		"boo",
+	} {
+		_, err := CidEncoderFromPath(badPath)
+		if err == nil {
+			t.Errorf("expected error extracting encoder from bad path: %s", badPath)
+		}
+	}
+}

core/commands/dag/dag.go

@@ -14,6 +14,7 @@ import (
 	cmds "gx/ipfs/QmWGm4AbZEbnmdgVTza52MSNpEmBdFVqzmAysRbjrRyGbH/go-ipfs-cmds"
 	files "gx/ipfs/QmXWZCd8jfaHmt4UDSnjKmGcrQMw95bDGWqEeVLVJjoANX/go-ipfs-files"
 	ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format"
+	cidenc "gx/ipfs/QmdPQx9fvN5ExVwMhRmh7YpCQJzJrFhd1AjVBwJmRMFJeX/go-cidutil/cidenc"
 	cmdkit "gx/ipfs/Qmde5VP1qUkyQXKCfmEUA7bP64V2HAptbJ7phuPp7jXWwg/go-ipfs-cmdkit"
 	mh "gx/ipfs/QmerPMzPk1mJVowm8KgmoknWa4yCYvvugMPsgWmDNUvDLW/go-multihash"
 )
@@ -144,7 +145,11 @@ into an object of the specified format.
 	Type: OutputObject{},
 	Encoders: cmds.EncoderMap{
 		cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *OutputObject) error {
-			fmt.Fprintln(w, out.Cid.String())
+			enc, err := cmdenv.GetLowLevelCidEncoder(req)
+			if err != nil {
+				return err
+			}
+			fmt.Fprintln(w, enc.Encode(out.Cid))
 			return nil
 		}),
 	},
@@ -227,7 +232,26 @@ var DagResolveCmd = &cmds.Command{
 	},
 	Encoders: cmds.EncoderMap{
 		cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *ResolveOutput) error {
-			p := out.Cid.String()
+			var (
+				enc cidenc.Encoder
+				err error
+			)
+			switch {
+			case !cmdenv.CidBaseDefined(req):
+				// Not specified, check the path.
+				enc, err = cmdenv.CidEncoderFromPath(req.Arguments[0])
+				if err == nil {
+					break
+				}
+				// Nope, fallback on the default.
+				fallthrough
+			default:
+				enc, err = cmdenv.GetLowLevelCidEncoder(req)
+				if err != nil {
+					return err
+				}
+			}
+			p := enc.Encode(out.Cid)
 			if out.RemPath != "" {
 				p = path.Join([]string{p, out.RemPath})
 			}

core/commands/files.go

@@ -17,13 +17,14 @@ import (
 	"gx/ipfs/QmP9eu5X5Ax8169jNWqAJcc42mdZgzLR1aKCEzqhNoBLKk/go-mfs"
 	"gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize"
 	ft "gx/ipfs/QmQXze9tG878pa4Euya4rrDpyTNX3kQe4dhCaBzBozGgpe/go-unixfs"
-	"gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid"
+	cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid"
 	dag "gx/ipfs/QmTQdH4848iTVCJmKXYyRiK72HufWTLYQQ8iN3JaQ8K1Hq/go-merkledag"
 	"gx/ipfs/QmWGm4AbZEbnmdgVTza52MSNpEmBdFVqzmAysRbjrRyGbH/go-ipfs-cmds"
 	bservice "gx/ipfs/QmYPZzd9VqmJDwxUnThfeSbV1Y5o53aVPDijTB7j7rS9Ep/go-blockservice"
 	"gx/ipfs/QmYZwey1thDTynSrvd6qQkX24UpTka6TFhQ2v569UpoqxD/go-ipfs-exchange-offline"
 	ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format"
 	logging "gx/ipfs/QmcuXC5cxs79ro2cUuHs4HQ2bkDLJUYokwL8aivcX6HW3C/go-log"
+	cidenc "gx/ipfs/QmdPQx9fvN5ExVwMhRmh7YpCQJzJrFhd1AjVBwJmRMFJeX/go-cidutil/cidenc"
 	"gx/ipfs/Qmde5VP1qUkyQXKCfmEUA7bP64V2HAptbJ7phuPp7jXWwg/go-ipfs-cmdkit"
 	mh "gx/ipfs/QmerPMzPk1mJVowm8KgmoknWa4yCYvvugMPsgWmDNUvDLW/go-multihash"
 )
@@ -136,6 +137,11 @@ var filesStatCmd = &cmds.Command{
 
 		withLocal, _ := req.Options[filesWithLocalOptionName].(bool)
 
+		enc, err := cmdenv.GetCidEncoder(req)
+		if err != nil {
+			return err
+		}
+
 		var dagserv ipld.DAGService
 		if withLocal {
 			// an offline DAGService will not fetch from the network
@@ -152,7 +158,7 @@ var filesStatCmd = &cmds.Command{
 			return err
 		}
 
-		o, err := statNode(nd)
+		o, err := statNode(nd, enc)
 		if err != nil {
 			return err
 		}
@@ -217,7 +223,7 @@ func statGetFormatOptions(req *cmds.Request) (string, error) {
 	}
 }
 
-func statNode(nd ipld.Node) (*statOutput, error) {
+func statNode(nd ipld.Node, enc cidenc.Encoder) (*statOutput, error) {
 	c := nd.Cid()
 
 	cumulsize, err := nd.Size()
@@ -243,15 +249,15 @@ func statNode(nd ipld.Node) (*statOutput, error) {
 		}
 
 		return &statOutput{
-			Hash:           c.String(),
+			Hash:           enc.Encode(c),
 			Blocks:         len(nd.Links()),
 			Size:           d.FileSize(),
 			CumulativeSize: cumulsize,
 			Type:           ndtype,
 		}, nil
 	case *dag.RawNode:
 		return &statOutput{
-			Hash:           c.String(),
+			Hash:           enc.Encode(c),
 			Blocks:         0,
 			Size:           cumulsize,
 			CumulativeSize: cumulsize,
@@ -433,6 +439,11 @@ Examples:
 
 		long, _ := req.Options[longOptionName].(bool)
 
+		enc, err := cmdenv.GetCidEncoder(req)
+		if err != nil {
+			return err
+		}
+
 		switch fsn := fsn.(type) {
 		case *mfs.Directory:
 			if !long {
@@ -470,7 +481,7 @@ Examples:
 				if err != nil {
 					return err
 				}
-				out.Entries[0].Hash = nd.Cid().String()
+				out.Entries[0].Hash = enc.Encode(nd.Cid())
 			}
 			return cmds.EmitOnce(res, out)
 		default: