rcmgr: the last charge

.circleci/main.yml

@@ -309,7 +309,7 @@ jobs:
       - run:
           name: Cloning
           command: |
-            git clone https://github.com/ipfs/go-ipfs-http-client.git
+            git clone https://github.com/ipfs/go-ipfs-http-client.git -b bump-for-rcmgr-last-push
             git -C go-ipfs-http-client log -1
       - restore_cache:
           keys:

.github/workflows/build.yml

@@ -149,6 +149,7 @@ jobs:
         with:
           repository: ipfs/go-ipfs-http-client
           path: go-ipfs-http-client
+          ref: bump-for-rcmgr-last-push
       - uses: protocol/cache-go-action@v1
         with:
           name: ${{ github.job }}

config/swarm.go

@@ -1,7 +1,5 @@
 package config
 
-import rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"
-
 type SwarmConfig struct {
 	// AddrFilters specifies a set libp2p addresses that we should never
 	// dial or receive connections from.
@@ -141,8 +139,8 @@ type ConnMgr struct {
 // <https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#readme>
 type ResourceMgr struct {
 	// Enables the Network Resource Manager feature, default to on.
-	Enabled Flag                      `json:",omitempty"`
-	Limits  *rcmgr.PartialLimitConfig `json:",omitempty"`
+	Enabled Flag        `json:",omitempty"`
+	Limits  swarmLimits `json:",omitempty"`
 
 	MaxMemory          *OptionalString  `json:",omitempty"`
 	MaxFileDescriptors *OptionalInteger `json:",omitempty"`

config/types.go

@@ -1,8 +1,10 @@
 package config
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 	"strings"
 	"time"
 )
@@ -412,3 +414,27 @@ func (p OptionalString) String() string {
 
 var _ json.Unmarshaler = (*OptionalInteger)(nil)
 var _ json.Marshaler = (*OptionalInteger)(nil)
+
+type swarmLimits struct{}
+
+var _ json.Unmarshaler = swarmLimits{}
+
+func (swarmLimits) UnmarshalJSON(b []byte) error {
+	d := json.NewDecoder(bytes.NewReader(b))
+	for {
+		switch tok, err := d.Token(); err {
+		case io.EOF:
+			return nil
+		case nil:
+			switch tok {
+			case json.Delim('{'), json.Delim('}'):
+				// accept empty objects
+				continue
+			}
+			//nolint
+			return fmt.Errorf("The Swarm.ResourceMgr.Limits configuration has been removed in Kubo 0.19 and should be empty or not present. To set custom libp2p limits, read https://github.com/ipfs/kubo/blob/master/docs/libp2p-resource-management.md#user-supplied-override-limits")
+		default:
+			return err
+		}
+	}
+}

core/commands/commands_test.go

@@ -248,13 +248,12 @@ func TestCommands(t *testing.T) {
 		"/swarm/filters",
 		"/swarm/filters/add",
 		"/swarm/filters/rm",
-		"/swarm/limit",
 		"/swarm/peers",
 		"/swarm/peering",
 		"/swarm/peering/add",
 		"/swarm/peering/ls",
 		"/swarm/peering/rm",
-		"/swarm/stats",
+		"/swarm/resources",
 		"/tar",
 		"/tar/add",
 		"/tar/cat",

core/commands/swarm.go

@@ -1,18 +1,18 @@
 package commands
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"path"
 	"sort"
+	"strconv"
 	"sync"
+	"text/tabwriter"
 	"time"
 
-	"github.com/ipfs/go-libipfs/files"
 	"github.com/ipfs/kubo/commands"
 	"github.com/ipfs/kubo/config"
 	"github.com/ipfs/kubo/core/commands/cmdenv"
@@ -57,8 +57,8 @@ ipfs peers in the internet.
 		"filters":    swarmFiltersCmd,
 		"peers":      swarmPeersCmd,
 		"peering":    swarmPeeringCmd,
-		"stats":      swarmStatsCmd, // libp2p Network Resource Manager
-		"limit":      swarmLimitCmd, // libp2p Network Resource Manager
+		"resources":  swarmResourcesCmd, // libp2p Network Resource Manager
+
 	},
 }
 
@@ -323,30 +323,15 @@ var swarmPeersCmd = &cmds.Command{
 	Type: connInfos{},
 }
 
-var swarmStatsCmd = &cmds.Command{
+var swarmResourcesCmd = &cmds.Command{
 	Status: cmds.Experimental,
 	Helptext: cmds.HelpText{
-		Tagline: "Report resource usage for a scope.",
-		LongDescription: `Report resource usage for a scope.
-The scope can be one of the following:
-- system        -- reports the system aggregate resource usage.
-- transient     -- reports the transient resource usage.
-- svc:<service> -- reports the resource usage of a specific service.
-- proto:<proto> -- reports the resource usage of a specific protocol.
-- peer:<peer>   -- reports the resource usage of a specific peer.
-- all           -- reports the resource usage for all currently active scopes.
-
-The output of this command is JSON.
-
-To see all resources that are close to hitting their respective limit, one can do something like:
-  ipfs swarm stats --min-used-limit-perc=90 all
+		Tagline: "Get a summary of all resources accounted for by the libp2p Resource Manager.",
+		LongDescription: `
+Get a summary of all resources accounted for by the libp2p Resource Manager.
+This includes the limits and the usage against those limits.
+This can output a human readable table and JSON encoding.
 `},
-	Arguments: []cmds.Argument{
-		cmds.StringArg("scope", true, false, "scope of the stat report"),
-	},
-	Options: []cmds.Option{
-		cmds.IntOption(swarmUsedResourcesPercentageName, "Only display resources that are using above the specified percentage of their respective limit"),
-	},
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
 		node, err := cmdenv.GetNode(env)
 		if err != nil {
@@ -357,128 +342,68 @@ To see all resources that are close to hitting their respective limit, one can d
 			return libp2p.ErrNoResourceMgr
 		}
 
-		if len(req.Arguments) != 1 {
-			return fmt.Errorf("must specify exactly one scope")
-		}
-
-		percentage, _ := req.Options[swarmUsedResourcesPercentageName].(int)
-		scope := req.Arguments[0]
-
-		if percentage != 0 && scope != "all" {
-			return fmt.Errorf("%q can only be used when scope is %q", swarmUsedResourcesPercentageName, "all")
-		}
-
-		result, err := libp2p.NetStat(node.ResourceManager, scope, percentage)
+		cfg, err := node.Repo.Config()
 		if err != nil {
 			return err
 		}
 
-		b := new(bytes.Buffer)
-		enc := json.NewEncoder(b)
-		err = enc.Encode(result)
+		userResourceOverrides, err := node.Repo.UserResourceOverrides()
 		if err != nil {
 			return err
 		}
-		return cmds.EmitOnce(res, b)
-	},
-	Encoders: cmds.EncoderMap{
-		cmds.Text: HumanJSONEncoder,
-	},
-}
-
-var swarmLimitCmd = &cmds.Command{
-	Status: cmds.Experimental,
-	Helptext: cmds.HelpText{
-		Tagline: "Get or set resource limits for a scope.",
-		LongDescription: `Get or set resource limits for a scope.
-The scope can be one of the following:
-- all           -- all limits actually being applied.
-- system        -- limits for the system aggregate resource usage.
-- transient     -- limits for the transient resource usage.
-- svc:<service> -- limits for the resource usage of a specific service.
-- proto:<proto> -- limits for the resource usage of a specific protocol.
-- peer:<peer>   -- limits for the resource usage of a specific peer.
-
-The output of this command is JSON.
-
-It is possible to use this command to inspect and tweak limits at runtime:
-
-	$ ipfs swarm limit system > limit.json
-	$ vi limit.json
-	$ ipfs swarm limit system limit.json
 
-Changes made via command line are persisted in the Swarm.ResourceMgr.Limits field of the $IPFS_PATH/config file.
-`},
-	Arguments: []cmds.Argument{
-		cmds.StringArg("scope", true, false, "scope of the limit"),
-		cmds.FileArg("limit.json", false, false, "limits to be set").EnableStdin(),
-	},
-	Options: []cmds.Option{
-		cmds.BoolOption(swarmResetLimitsOptionName, "reset limit to default"),
-	},
-	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
-		node, err := cmdenv.GetNode(env)
+		// FIXME: we shouldn't recompute limits, either save them or load them from libp2p (https://github.com/libp2p/go-libp2p/issues/2166)
+		limitConfig, _, err := libp2p.LimitConfig(cfg.Swarm, userResourceOverrides)
 		if err != nil {
 			return err
 		}
 
-		if node.ResourceManager == nil {
+		rapi, ok := node.ResourceManager.(rcmgr.ResourceManagerState)
+		if !ok { // NullResourceManager
 			return libp2p.ErrNoResourceMgr
 		}
 
-		scope := req.Arguments[0]
-
-		//  set scope limit to new values (when limit.json is passed as a second arg)
-		if req.Files != nil {
-			var newLimit rcmgr.ResourceLimits
-			it := req.Files.Entries()
-			if it.Next() {
-				file := files.FileFromEntry(it)
-				if file == nil {
-					return errors.New("expected a JSON file")
-				}
-
-				r := io.LimitReader(file, 32*1024*1024) // 32MiB
-
-				if err := json.NewDecoder(r).Decode(&newLimit); err != nil {
-					return fmt.Errorf("decoding JSON as ResourceMgrScopeConfig: %w", err)
+		return cmds.EmitOnce(res, libp2p.MergeLimitsAndStatsIntoLimitsConfigAndUsage(limitConfig, rapi.Stat()))
+	},
+	Encoders: cmds.EncoderMap{
+		cmds.JSON: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, limitsAndUsage libp2p.LimitsConfigAndUsage) error {
+			return json.NewEncoder(w).Encode(limitsAndUsage)
+		}),
+		cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, limitsAndUsage libp2p.LimitsConfigAndUsage) error {
+			tw := tabwriter.NewWriter(w, 20, 8, 0, '\t', 0)
+			defer tw.Flush()
+
+			fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t\n", "Scope", "Limit Name", "Limit Value", "Limit Usage Amount", "Limit Usage Percent")
+			for _, ri := range libp2p.LimitConfigsToInfo(limitsAndUsage) {
+				var limit, percentage string
+				switch ri.LimitValue {
+				case rcmgr.Unlimited64:
+					limit = "unlimited"
+					percentage = "n/a"
+				case rcmgr.BlockAllLimit64:
+					limit = "blockAll"
+					percentage = "n/a"
+				default:
+					limit = strconv.FormatInt(int64(ri.LimitValue), 10)
+					if ri.CurrentUsage == 0 {
+						percentage = "0%"
+					} else {
+						percentage = strconv.FormatFloat(float64(ri.CurrentUsage)/float64(ri.LimitValue)*100, 'f', 1, 64) + "%"
+					}
 				}
-				return libp2p.NetSetLimit(node.ResourceManager, node.Repo, scope, newLimit)
+				fmt.Fprintf(tw, "%s\t%s\t%s\t%d\t%s\t\n",
+					ri.ScopeName,
+					ri.LimitName,
+					limit,
+					ri.CurrentUsage,
+					percentage,
+				)
 			}
-			if err := it.Err(); err != nil {
-				return fmt.Errorf("error opening limit JSON file: %w", err)
-			}
-		}
-
-		var result interface{}
-		switch _, reset := req.Options[swarmResetLimitsOptionName]; {
-		case reset:
-			result, err = libp2p.NetResetLimit(node.ResourceManager, node.Repo, scope)
-		case scope == "all":
-			result, err = libp2p.NetLimitAll(node.ResourceManager)
-		default:
-			// get scope limit
-			result, err = libp2p.NetLimit(node.ResourceManager, scope)
-		}
-		if err != nil {
-			return err
-		}
 
-		if base, ok := result.(rcmgr.BaseLimit); ok {
-			result = base.ToResourceLimits()
-		}
-
-		b := new(bytes.Buffer)
-		enc := json.NewEncoder(b)
-		err = enc.Encode(result)
-		if err != nil {
-			return err
-		}
-		return cmds.EmitOnce(res, b)
-	},
-	Encoders: cmds.EncoderMap{
-		cmds.Text: HumanJSONEncoder,
+			return nil
+		}),
 	},
+	Type: libp2p.LimitsConfigAndUsage{},
 }
 
 type streamInfo struct {