config: introduce Import section

Author: hacdias

config/config.go

@@ -36,6 +36,7 @@ type Config struct {
 	Experimental Experiments
 	Plugins      Plugins
 	Pinning      Pinning
+	Import       Import
 
 	Internal Internal // experimental/unstable options
 }

config/import.go

@@ -0,0 +1,17 @@
+package config
+
+const (
+	DefaultCidVersion      = 0
+	DefaultUnixFSRawLeaves = false
+	DefaultUnixFSChunker   = "size-262144"
+	DefaultHashFunction    = "sha2-256"
+)
+
+// Import configures the default options for ingesting data. This affects commands
+// that ingest data, such as 'ipfs add', 'ipfs dag put, 'ipfs block put', 'ipfs files write'.
+type Import struct {
+	CidVersion      OptionalInteger
+	UnixFSRawLeaves Flag
+	UnixFSChunker   OptionalString
+	HashFunction    OptionalString
+}

core/commands/add.go

@@ -8,6 +8,7 @@ import (
 	gopath "path"
 	"strings"
 
+	"github.com/ipfs/kubo/config"
 	"github.com/ipfs/kubo/core/commands/cmdenv"
 
 	"github.com/cheggaaa/pb"
@@ -155,12 +156,12 @@ See 'dag export' and 'dag import' for more information.
 		cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."),
 		cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."),
 		cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."),
-		cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash").WithDefault("size-262144"),
+		cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash"),
 		cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes."),
 		cmds.BoolOption(noCopyOptionName, "Add the file using filestore. Implies raw-leaves. (experimental)"),
 		cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"),
 		cmds.IntOption(cidVersionOptionName, "CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. Passing version 1 will cause the raw-leaves option to default to true."),
-		cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)").WithDefault("sha2-256"),
+		cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)"),
 		cmds.BoolOption(inlineOptionName, "Inline small blocks into CIDs. (experimental)"),
 		cmds.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32),
 		cmds.BoolOption(pinOptionName, "Pin locally to protect added files from garbage collection.").WithDefault(true),
@@ -191,6 +192,16 @@ See 'dag export' and 'dag import' for more information.
 			return err
 		}
 
+		nd, err := cmdenv.GetNode(env)
+		if err != nil {
+			return err
+		}
+
+		cfg, err := nd.Repo.Config()
+		if err != nil {
+			return err
+		}
+
 		progress, _ := req.Options[progressOptionName].(bool)
 		trickle, _ := req.Options[trickleOptionName].(bool)
 		wrap, _ := req.Options[wrapOptionName].(bool)
@@ -207,6 +218,24 @@ See 'dag export' and 'dag import' for more information.
 		inlineLimit, _ := req.Options[inlineLimitOptionName].(int)
 		toFilesStr, toFilesSet := req.Options[toFilesOptionName].(string)
 
+		if chunker == "" {
+			chunker = cfg.Import.UnixFSChunker.WithDefault(config.DefaultUnixFSChunker)
+		}
+
+		if hashFunStr == "" {
+			hashFunStr = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction)
+		}
+
+		if !cidVerSet && !cfg.Import.CidVersion.IsDefault() {
+			cidVerSet = true
+			cidVer = int(cfg.Import.CidVersion.WithDefault(config.DefaultCidVersion))
+		}
+
+		if !rbset && cfg.Import.UnixFSRawLeaves != config.Default {
+			rbset = true
+			rawblks = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves)
+		}
+
 		if onlyHash && toFilesSet {
 			return fmt.Errorf("%s and %s options are not compatible", onlyHashOptionName, toFilesOptionName)
 		}

core/commands/block.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/ipfs/boxo/files"
 
+	"github.com/ipfs/kubo/config"
 	cmdenv "github.com/ipfs/kubo/core/commands/cmdenv"
 	"github.com/ipfs/kubo/core/commands/cmdutils"
 
@@ -153,7 +154,7 @@ only for backward compatibility when a legacy CIDv0 is required (--format=v0).
 	},
 	Options: []cmds.Option{
 		cmds.StringOption(blockCidCodecOptionName, "Multicodec to use in returned CID").WithDefault("raw"),
-		cmds.StringOption(mhtypeOptionName, "Multihash hash function").WithDefault("sha2-256"),
+		cmds.StringOption(mhtypeOptionName, "Multihash hash function"),
 		cmds.IntOption(mhlenOptionName, "Multihash hash length").WithDefault(-1),
 		cmds.BoolOption(pinOptionName, "Pin added blocks recursively").WithDefault(false),
 		cmdutils.AllowBigBlockOption,
@@ -165,7 +166,21 @@ only for backward compatibility when a legacy CIDv0 is required (--format=v0).
 			return err
 		}
 
+		nd, err := cmdenv.GetNode(env)
+		if err != nil {
+			return err
+		}
+
+		cfg, err := nd.Repo.Config()
+		if err != nil {
+			return err
+		}
+
 		mhtype, _ := req.Options[mhtypeOptionName].(string)
+		if mhtype == "" {
+			mhtype = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction)
+		}
+
 		mhtval, ok := mh.Names[mhtype]
 		if !ok {
 			return fmt.Errorf("unrecognized multihash function: %s", mhtype)

core/commands/dag/dag.go

@@ -87,7 +87,7 @@ into an object of the specified format.
 		cmds.StringOption("store-codec", "Codec that the stored object will be encoded with").WithDefault("dag-cbor"),
 		cmds.StringOption("input-codec", "Codec that the input object is encoded in").WithDefault("dag-json"),
 		cmds.BoolOption("pin", "Pin this object when adding."),
-		cmds.StringOption("hash", "Hash function to use").WithDefault("sha2-256"),
+		cmds.StringOption("hash", "Hash function to use"),
 		cmdutils.AllowBigBlockOption,
 	},
 	Run:  dagPut,

core/commands/dag/put.go

@@ -7,6 +7,7 @@ import (
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	ipldlegacy "github.com/ipfs/go-ipld-legacy"
+	"github.com/ipfs/kubo/config"
 	"github.com/ipfs/kubo/core/commands/cmdenv"
 	"github.com/ipfs/kubo/core/commands/cmdutils"
 	"github.com/ipld/go-ipld-prime/multicodec"
@@ -32,11 +33,25 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e
 		return err
 	}
 
+	nd, err := cmdenv.GetNode(env)
+	if err != nil {
+		return err
+	}
+
+	cfg, err := nd.Repo.Config()
+	if err != nil {
+		return err
+	}
+
 	inputCodec, _ := req.Options["input-codec"].(string)
 	storeCodec, _ := req.Options["store-codec"].(string)
 	hash, _ := req.Options["hash"].(string)
 	dopin, _ := req.Options["pin"].(bool)
 
+	if hash == "" {
+		hash = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction)
+	}
+
 	var icodec mc.Code
 	if err := icodec.Set(inputCodec); err != nil {
 		return err

core/commands/files.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/ipfs/kubo/config"
 	"github.com/ipfs/kubo/core"
 	"github.com/ipfs/kubo/core/commands/cmdenv"
 
@@ -802,18 +803,28 @@ See '--to-files' in 'ipfs add --help' for more information.
 			return err
 		}
 
+		nd, err := cmdenv.GetNode(env)
+		if err != nil {
+			return err
+		}
+
+		cfg, err := nd.Repo.Config()
+		if err != nil {
+			return err
+		}
+
 		create, _ := req.Options[filesCreateOptionName].(bool)
 		mkParents, _ := req.Options[filesParentsOptionName].(bool)
 		trunc, _ := req.Options[filesTruncateOptionName].(bool)
 		flush, _ := req.Options[filesFlushOptionName].(bool)
 		rawLeaves, rawLeavesDef := req.Options[filesRawLeavesOptionName].(bool)
 
-		prefix, err := getPrefixNew(req)
-		if err != nil {
-			return err
+		if !rawLeavesDef && cfg.Import.UnixFSRawLeaves != config.Default {
+			rawLeavesDef = true
+			rawLeaves = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves)
 		}
 
-		nd, err := cmdenv.GetNode(env)
+		prefix, err := getPrefixNew(req)
 		if err != nil {
 			return err
 		}

docs/changelogs/v0.29.md

@@ -13,6 +13,10 @@
 
 ### 🔦 Highlights
 
+#### Global configuration for data ingestion
+
+A new configuration section, `Import`, was introduced. This section allows to override the default values of options used across several commands that do data ingestion. Read more in the [config documentation](../config.md).
+
 ### 📝 Changelog
 
 ### 👨‍👩‍👧‍👦 Contributors

docs/config.md

@@ -175,6 +175,11 @@ config file at runtime.
   - [`DNS`](#dns)
     - [`DNS.Resolvers`](#dnsresolvers)
     - [`DNS.MaxCacheTTL`](#dnsmaxcachettl)
+  - [`Import`](#import)
+    - [`Import.CidVersion`](#importcidversion)
+    - [`Import.UnixFSRawLeaves`](#importunixfsrawleaves)
+    - [`Import.UnixFSChunker`](#importunixfschunker)
+    - [`Import.HashFunction`](#importhashfunction)
 
 ## Profiles
 
@@ -2377,3 +2382,42 @@ Note: this does NOT work with Go's default DNS resolver. To make this a global s
 Default: Respect DNS Response TTL
 
 Type: `optionalDuration`
+
+## `Import`
+
+Options to configure the default options used for ingesting data, in commands such as `ipfs add` or `ipfs block put`. All affected commands are detailed per option.
+
+Note that using flags will override the options defined here.
+
+### `Import.CidVersion`
+
+The default CID version. Commands affected: `ipfs add`.
+
+Default: `0`
+
+Type: `optionalInteger`
+
+### `Import.UnixFSRawLeaves`
+
+The default UnixFS raw leaves option. Commands affected: `ipfs add`, `ipfs files write`.
+
+Default: `false`
+
+Type: `flag`
+
+### `Import.UnixFSChunker`
+
+The default UnixFS chunker. Commands affected: `ipfs add`.
+
+Default: `size-262144`
+
+Type: `optionalString`
+
+
+### `Import.HashFunction`
+
+The default hash function. Commands affected: `ipfs add`, `ipfs block put`, `ipfs dag put`.
+
+Default: `sha2-256`
+
+Type: `optionalString`

test/cli/add_test.go

@@ -0,0 +1,81 @@
+package cli
+
+import (
+	"testing"
+
+	"github.com/ipfs/kubo/config"
+	"github.com/ipfs/kubo/test/cli/harness"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAdd(t *testing.T) {
+	t.Parallel()
+
+	var (
+		shortString            = "hello world"
+		shortStringCidV0       = "Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD"
+		shortStringCidV1       = "bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e"
+		shortStringCidV1Sha512 = "bafkrgqbqt3gerhas23vuzrapkdeqf4vu2dwxp3srdj6hvg6nhsug2tgyn6mj3u23yx7utftq3i2ckw2fwdh5qmhid5qf3t35yvkc5e5ottlw6"
+	)
+
+	t.Run("output cid version: default (CIDv0)", func(t *testing.T) {
+		t.Parallel()
+		node := harness.NewT(t).NewNode().Init().StartDaemon()
+		defer node.StopDaemon()
+
+		cidStr := node.IPFSAddStr(shortString)
+		require.Equal(t, shortStringCidV0, cidStr)
+	})
+
+	t.Run("output cid version: follows configuration (CIDv0)", func(t *testing.T) {
+		t.Parallel()
+		node := harness.NewT(t).NewNode().Init()
+		node.UpdateConfig(func(cfg *config.Config) {
+			cfg.Import.CidVersion = *config.NewOptionalInteger(0)
+		})
+		node.StartDaemon()
+		defer node.StopDaemon()
+
+		cidStr := node.IPFSAddStr(shortString)
+		require.Equal(t, shortStringCidV0, cidStr)
+	})
+
+	t.Run("output cid version: follows configuration (hash function)", func(t *testing.T) {
+		t.Parallel()
+		node := harness.NewT(t).NewNode().Init()
+		node.UpdateConfig(func(cfg *config.Config) {
+			cfg.Import.HashFunction = *config.NewOptionalString("sha2-512")
+		})
+		node.StartDaemon()
+		defer node.StopDaemon()
+
+		cidStr := node.IPFSAddStr(shortString)
+		require.Equal(t, shortStringCidV1Sha512, cidStr)
+	})
+
+	t.Run("output cid version: follows configuration (CIDv1)", func(t *testing.T) {
+		t.Parallel()
+		node := harness.NewT(t).NewNode().Init()
+		node.UpdateConfig(func(cfg *config.Config) {
+			cfg.Import.CidVersion = *config.NewOptionalInteger(1)
+		})
+		node.StartDaemon()
+		defer node.StopDaemon()
+
+		cidStr := node.IPFSAddStr(shortString)
+		require.Equal(t, shortStringCidV1, cidStr)
+	})
+
+	t.Run("output cid version: flag overrides configuration", func(t *testing.T) {
+		t.Parallel()
+		node := harness.NewT(t).NewNode().Init()
+		node.UpdateConfig(func(cfg *config.Config) {
+			cfg.Import.CidVersion = *config.NewOptionalInteger(1)
+		})
+		node.StartDaemon()
+		defer node.StopDaemon()
+
+		cidStr := node.IPFSAddStr(shortString, "--cid-version", "0")
+		require.Equal(t, shortStringCidV0, cidStr)
+	})
+}