diff --git a/go/consensus/cometbft/cometbft.go b/go/consensus/cometbft/cometbft.go index 313a5ea2b4e..b4f3881eb0a 100644 --- a/go/consensus/cometbft/cometbft.go +++ b/go/consensus/cometbft/cometbft.go @@ -31,6 +31,7 @@ func New( genesis genesisAPI.Provider, doc *genesisAPI.Document, p2p p2pAPI.Service, + metricsEnabled bool, ) (consensusAPI.Service, error) { genesisDoc, err := api.GetCometBFTGenesisDocument(doc) if err != nil { @@ -39,7 +40,7 @@ func New( switch config.GlobalConfig.Mode { case config.ModeArchive: - node, err := createArchiveNode(ctx, dataDir, identity, genesis, doc, genesisDoc) + node, err := createArchiveNode(ctx, dataDir, identity, genesis, doc, genesisDoc, metricsEnabled) if err != nil { return nil, fmt.Errorf("failed to create archive node: %w", err) } @@ -51,7 +52,7 @@ func New( } return node, nil default: - node, err := createFullNode(ctx, dataDir, identity, genesis, doc, genesisDoc, upgrader, p2p) + node, err := createFullNode(ctx, dataDir, identity, genesis, doc, genesisDoc, upgrader, p2p, metricsEnabled) if err != nil { return nil, fmt.Errorf("failed to create full node: %w", err) } @@ -66,9 +67,10 @@ func createArchiveNode( genesis genesisAPI.Provider, doc *genesisAPI.Document, genesisDoc *cmttypes.GenesisDoc, + metricsEnabled bool, ) (consensusAPI.Service, error) { cfg := full.ArchiveConfig{ - CommonConfig: createCommonConfig(dataDir, identity, genesis, doc, genesisDoc), + CommonConfig: createCommonConfig(dataDir, identity, genesis, doc, genesisDoc, metricsEnabled), } return full.NewArchive(ctx, cfg) @@ -83,9 +85,10 @@ func createFullNode( genesisDoc *cmttypes.GenesisDoc, upgrader upgradeAPI.Backend, p2p p2pAPI.Service, + metricsEnabled bool, ) (consensusAPI.Service, error) { cfg := full.Config{ - CommonConfig: createCommonConfig(dataDir, identity, genesis, doc, genesisDoc), + CommonConfig: createCommonConfig(dataDir, identity, genesis, doc, genesisDoc, metricsEnabled), TimeoutCommit: doc.Consensus.Parameters.TimeoutCommit, EmptyBlockInterval: doc.Consensus.Parameters.EmptyBlockInterval, SkipTimeoutCommit: doc.Consensus.Parameters.SkipTimeoutCommit, @@ -193,6 +196,7 @@ func createCommonConfig( genesis genesisAPI.Provider, doc *genesisAPI.Document, genesisDoc *cmttypes.GenesisDoc, + metricsEnabled bool, ) full.CommonConfig { return full.CommonConfig{ DataDir: dataDir, @@ -205,6 +209,7 @@ func createCommonConfig( BaseEpoch: doc.Beacon.Base, BaseHeight: doc.Height, PublicKeyBlacklist: doc.Consensus.Parameters.PublicKeyBlacklist, + MetricsEnabled: metricsEnabled, } } diff --git a/go/consensus/cometbft/full/common.go b/go/consensus/cometbft/full/common.go index 496c19eee6b..0064f306734 100644 --- a/go/consensus/cometbft/full/common.go +++ b/go/consensus/cometbft/full/common.go @@ -59,7 +59,6 @@ import ( governanceAPI "github.com/oasisprotocol/oasis-core/go/governance/api" keymanagerAPI "github.com/oasisprotocol/oasis-core/go/keymanager/api" cmbackground "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/background" - cmmetrics "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/metrics" "github.com/oasisprotocol/oasis-core/go/registry" registryAPI "github.com/oasisprotocol/oasis-core/go/registry/api" roothashAPI "github.com/oasisprotocol/oasis-core/go/roothash/api" @@ -101,6 +100,8 @@ type CommonConfig struct { BaseHeight int64 // PublicKeyBlacklist is the network-wide public key blacklist. PublicKeyBlacklist []signature.PublicKey + // MetricsEnabled is true if prometheus metrics are enabled. + MetricsEnabled bool } // commonNode implements the common CometBFT node functionality shared between @@ -149,6 +150,8 @@ type commonNode struct { startedCh chan struct{} parentNode consensusAPI.Backend + + metricsEnabled bool } func (n *commonNode) initialized() bool { @@ -304,7 +307,7 @@ func (n *commonNode) initialize() error { } // Start metrics. - if cmmetrics.Enabled() { + if n.metricsEnabled { rmu := registry.NewMetricsUpdater(n.ctx, n.registry) n.svcMgr.RegisterCleanupOnly(rmu, "registry metrics updater") } @@ -909,6 +912,7 @@ func newCommonNode(ctx context.Context, cfg CommonConfig) *commonNode { svcMgr: cmbackground.NewServiceManager(logging.GetLogger("cometbft/servicemanager")), dbCloser: db.NewCloser(), startedCh: make(chan struct{}), + metricsEnabled: cfg.MetricsEnabled, } } diff --git a/go/consensus/cometbft/full/full.go b/go/consensus/cometbft/full/full.go index fbed34770d6..b0e835cc4fb 100644 --- a/go/consensus/cometbft/full/full.go +++ b/go/consensus/cometbft/full/full.go @@ -49,7 +49,6 @@ import ( "github.com/oasisprotocol/oasis-core/go/consensus/metrics" "github.com/oasisprotocol/oasis-core/go/consensus/pricediscovery" cmflags "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/flags" - cmmetrics "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/metrics" p2pAPI "github.com/oasisprotocol/oasis-core/go/p2p/api" registryAPI "github.com/oasisprotocol/oasis-core/go/registry/api" stakingAPI "github.com/oasisprotocol/oasis-core/go/staking/api" @@ -173,7 +172,7 @@ func (t *fullService) Start() error { // Start block notifier. go t.blockNotifierWorker() // Optionally start metrics updater. - if cmmetrics.Enabled() { + if t.commonNode.metricsEnabled { go t.metrics() } case false: diff --git a/go/oasis-node/cmd/common/metrics/config/config.go b/go/oasis-node/cmd/common/metrics/config/config.go index 2d005335b17..b5379d216f1 100644 --- a/go/oasis-node/cmd/common/metrics/config/config.go +++ b/go/oasis-node/cmd/common/metrics/config/config.go @@ -1,4 +1,4 @@ -// Package config implements global metrics configuration options. +// Package config implements metrics configuration options. package config import ( diff --git a/go/oasis-node/cmd/common/metrics/metrics.go b/go/oasis-node/cmd/common/metrics/metrics.go index 981c40719c1..0da42295f25 100644 --- a/go/oasis-node/cmd/common/metrics/metrics.go +++ b/go/oasis-node/cmd/common/metrics/metrics.go @@ -17,8 +17,8 @@ import ( "github.com/oasisprotocol/oasis-core/go/common/service" "github.com/oasisprotocol/oasis-core/go/common/version" - "github.com/oasisprotocol/oasis-core/go/config" "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/flags" + "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/metrics/config" "github.com/oasisprotocol/oasis-core/go/oasis-test-runner/env" ) @@ -107,8 +107,8 @@ func (s *pullService) Cleanup() { } } -func newPullService() (service.BackgroundService, error) { - addr := config.GlobalConfig.Metrics.Address +func newPullService(cfg *config.Config) (service.BackgroundService, error) { + addr := cfg.Address svc := *service.NewBaseBackgroundService("metrics") @@ -127,7 +127,7 @@ func newPullService() (service.BackgroundService, error) { ln: ln, s: &http.Server{Handler: promhttp.Handler(), ReadTimeout: 5 * time.Second}, errCh: make(chan error), - rsvc: newResourceService(config.GlobalConfig.Metrics.Interval), + rsvc: newResourceService(cfg.Interval), }, nil } @@ -222,14 +222,14 @@ func (s *pushService) initPusher(isReinit bool) { s.pusher = pusher } -func newPushService() (service.BackgroundService, error) { +func newPushService(cfg *config.Config) (service.BackgroundService, error) { svc := &pushService{ BaseBackgroundService: *service.NewBaseBackgroundService("metrics"), - addr: config.GlobalConfig.Metrics.Address, - jobName: config.GlobalConfig.Metrics.JobName, - labels: config.GlobalConfig.Metrics.Labels, - interval: config.GlobalConfig.Metrics.Interval, - rsvc: newResourceService(config.GlobalConfig.Metrics.Interval), + addr: cfg.Address, + jobName: cfg.JobName, + labels: cfg.Labels, + interval: cfg.Interval, + rsvc: newResourceService(cfg.Interval), stopCh: make(chan struct{}), quitCh: make(chan struct{}), } @@ -247,24 +247,25 @@ func newPushService() (service.BackgroundService, error) { } // New constructs a new metrics service. -func New() (service.BackgroundService, error) { - mode := strings.ToLower(config.GlobalConfig.Metrics.Mode) +func New(cfg *config.Config) (service.BackgroundService, error) { + mode := strings.ToLower(cfg.Mode) switch mode { case MetricsModeNone: return newStubService() case MetricsModePull: - return newPullService() + return newPullService(cfg) default: if mode == MetricsModePush && flags.DebugDontBlameOasis() { - return newPushService() + return newPushService(cfg) } return nil, fmt.Errorf("metrics: unsupported mode: '%v'", mode) } } // Enabled returns if metrics are enabled. -func Enabled() bool { - return config.GlobalConfig.Metrics.Mode != MetricsModeNone +func Enabled(mode string) bool { + mode = strings.ToLower(mode) + return mode != MetricsModeNone } // EscapeLabelCharacters replaces invalid prometheus label name characters with "_". @@ -289,7 +290,7 @@ func GetDefaultPushLabels(ti *env.ScenarioInstanceInfo) map[string]string { labels[EscapeLabelCharacters(f.Name)] = f.Value.String() }) // Override any labels passed to oasis-test-runner via CLI. - for k, v := range config.GlobalConfig.Metrics.Labels { + for k, v := range config.DefaultConfig().Labels { // TODO explain in the commit why this is safe. labels[k] = v } diff --git a/go/oasis-node/cmd/debug/byzantine/beacon_vrf.go b/go/oasis-node/cmd/debug/byzantine/beacon_vrf.go index 8f10b028b63..9a3c132af1a 100644 --- a/go/oasis-node/cmd/debug/byzantine/beacon_vrf.go +++ b/go/oasis-node/cmd/debug/byzantine/beacon_vrf.go @@ -14,6 +14,7 @@ import ( "github.com/oasisprotocol/oasis-core/go/common/crypto/signature" "github.com/oasisprotocol/oasis-core/go/common/logging" "github.com/oasisprotocol/oasis-core/go/common/node" + "github.com/oasisprotocol/oasis-core/go/config" consensus "github.com/oasisprotocol/oasis-core/go/consensus/api" "github.com/oasisprotocol/oasis-core/go/consensus/api/transaction" scheduler "github.com/oasisprotocol/oasis-core/go/scheduler/api" @@ -81,7 +82,9 @@ func doVRFBeaconScenario(*cobra.Command, []string) { } round := uint64(3) - b, err := initializeAndRegisterByzantineNode(runtimeID, node.RoleValidator, scheduler.RoleInvalid, false, true, round) + // For every command where applicable you will have to parse yaml config?? + cfg := &config.GlobalConfig + b, err := initializeAndRegisterByzantineNode(cfg, runtimeID, node.RoleValidator, scheduler.RoleInvalid, false, true, round) if err != nil { panic(fmt.Sprintf("error initializing node: %+v", err)) } diff --git a/go/oasis-node/cmd/debug/byzantine/byzantine.go b/go/oasis-node/cmd/debug/byzantine/byzantine.go index 2be4c068805..8b71e956685 100644 --- a/go/oasis-node/cmd/debug/byzantine/byzantine.go +++ b/go/oasis-node/cmd/debug/byzantine/byzantine.go @@ -16,6 +16,7 @@ import ( "github.com/oasisprotocol/oasis-core/go/common/node" "github.com/oasisprotocol/oasis-core/go/common/sgx/ias" "github.com/oasisprotocol/oasis-core/go/common/sgx/pcs" + "github.com/oasisprotocol/oasis-core/go/config" "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/flags" "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/grpc" "github.com/oasisprotocol/oasis-core/go/roothash/api/block" @@ -147,7 +148,10 @@ func doExecutorScenario(*cobra.Command, []string) { //nolint: gocyclo round := uint64(3) isTxScheduler := viper.GetBool(CfgPrimarySchedulerExpected) + // For every command where applicable you will have to parse yaml config?? + cfg := &config.GlobalConfig b, err := initializeAndRegisterByzantineNode( + cfg, runtimeID, node.RoleComputeWorker, scheduler.RoleWorker, diff --git a/go/oasis-node/cmd/debug/byzantine/cometbft.go b/go/oasis-node/cmd/debug/byzantine/cometbft.go index 1748955416c..0c0b038895e 100644 --- a/go/oasis-node/cmd/debug/byzantine/cometbft.go +++ b/go/oasis-node/cmd/debug/byzantine/cometbft.go @@ -26,13 +26,13 @@ func newHonestCometBFT(genesis genesis.Provider, genesisDoc *genesis.Document) * } } -func (ht *honestCometBFT) start(id *identity.Identity, dataDir string) error { +func (ht *honestCometBFT) start(id *identity.Identity, dataDir string, metricsEnabled bool) error { if ht.service != nil { return fmt.Errorf("honest CometBFT service already started") } var err error - ht.service, err = cometbft.New(context.Background(), dataDir, id, upgrade.NewDummyUpgradeManager(), ht.genesis, ht.genesisDoc, p2p.NewNop()) + ht.service, err = cometbft.New(context.Background(), dataDir, id, upgrade.NewDummyUpgradeManager(), ht.genesis, ht.genesisDoc, p2p.NewNop(), metricsEnabled) if err != nil { return fmt.Errorf("cometbft New: %w", err) } diff --git a/go/oasis-node/cmd/debug/byzantine/node.go b/go/oasis-node/cmd/debug/byzantine/node.go index 447def2d766..03bd6bb3618 100644 --- a/go/oasis-node/cmd/debug/byzantine/node.go +++ b/go/oasis-node/cmd/debug/byzantine/node.go @@ -13,10 +13,12 @@ import ( "github.com/oasisprotocol/oasis-core/go/common/identity" "github.com/oasisprotocol/oasis-core/go/common/logging" "github.com/oasisprotocol/oasis-core/go/common/node" + "github.com/oasisprotocol/oasis-core/go/config" consensus "github.com/oasisprotocol/oasis-core/go/consensus/api" genesis "github.com/oasisprotocol/oasis-core/go/genesis/file" cmdCommon "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common" cmdFlags "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/flags" + "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/metrics" "github.com/oasisprotocol/oasis-core/go/roothash/api/block" "github.com/oasisprotocol/oasis-core/go/roothash/api/commitment" scheduler "github.com/oasisprotocol/oasis-core/go/scheduler/api" @@ -96,6 +98,7 @@ func (b *byzantine) receiveAndScheduleTransactions(ctx context.Context, cbc *com } func initializeAndRegisterByzantineNode( + cfg *config.Config, runtimeID common.Namespace, nodeRoles node.RolesMask, expectedExecutorRole scheduler.Role, @@ -139,13 +142,14 @@ func initializeAndRegisterByzantineNode( // Setup CometBFT. b.cometbft = newHonestCometBFT(genesis, genesisDoc) - if err = b.cometbft.start(b.identity, cmdCommon.DataDir()); err != nil { + metricsEnabled := metrics.Enabled(cfg.Metrics.Mode) + if err = b.cometbft.start(b.identity, cmdCommon.DataDir(), metricsEnabled); err != nil { return nil, fmt.Errorf("node cometbft start failed: %w", err) } // Setup P2P. b.p2p = newP2PHandle() - if err = b.p2p.start(b.cometbft, b.identity, b.chainContext, b.runtimeID); err != nil { + if err = b.p2p.start(cfg, b.cometbft, b.identity, b.chainContext, b.runtimeID); err != nil { return nil, fmt.Errorf("P2P start failed: %w", err) } diff --git a/go/oasis-node/cmd/debug/byzantine/p2p.go b/go/oasis-node/cmd/debug/byzantine/p2p.go index 26e450b95ba..90e646b4113 100644 --- a/go/oasis-node/cmd/debug/byzantine/p2p.go +++ b/go/oasis-node/cmd/debug/byzantine/p2p.go @@ -8,6 +8,7 @@ import ( "github.com/oasisprotocol/oasis-core/go/common/cbor" "github.com/oasisprotocol/oasis-core/go/common/crypto/signature" "github.com/oasisprotocol/oasis-core/go/common/identity" + "github.com/oasisprotocol/oasis-core/go/config" "github.com/oasisprotocol/oasis-core/go/p2p" p2pAPI "github.com/oasisprotocol/oasis-core/go/p2p/api" "github.com/oasisprotocol/oasis-core/go/p2p/protocol" @@ -91,13 +92,17 @@ func (h *committeeMsgHandler) HandleMessage(_ context.Context, peerID signature. return <-responseCh } -func (ph *p2pHandle) start(ht *honestCometBFT, id *identity.Identity, chainContext string, runtimeID common.Namespace) error { +func (ph *p2pHandle) start(yamlCfg *config.Config, ht *honestCometBFT, id *identity.Identity, chainContext string, runtimeID common.Namespace) error { if ph.service != nil { return fmt.Errorf("P2P service already started") } + var cfg p2p.Config + if err := cfg.Load(&yamlCfg.P2P); err != nil { + return fmt.Errorf("failed to parse p2p config %w", err) + } var err error - ph.service, err = p2p.New(id, chainContext, nil) + ph.service, err = p2p.New(&cfg, id, chainContext, nil) if err != nil { return fmt.Errorf("P2P service New: %w", err) } diff --git a/go/oasis-node/cmd/ias/proxy.go b/go/oasis-node/cmd/ias/proxy.go index 723fb1d2b4d..1e313cce244 100644 --- a/go/oasis-node/cmd/ias/proxy.go +++ b/go/oasis-node/cmd/ias/proxy.go @@ -18,6 +18,7 @@ import ( "github.com/oasisprotocol/oasis-core/go/common/grpc" "github.com/oasisprotocol/oasis-core/go/common/logging" cmnIAS "github.com/oasisprotocol/oasis-core/go/common/sgx/ias" + "github.com/oasisprotocol/oasis-core/go/config" ias "github.com/oasisprotocol/oasis-core/go/ias/api" iasHTTP "github.com/oasisprotocol/oasis-core/go/ias/http" iasProxy "github.com/oasisprotocol/oasis-core/go/ias/proxy" @@ -152,8 +153,10 @@ func doProxy(cmd *cobra.Command, _ []string) { } env.svcMgr.Register(env.grpcSrv) + // For every command where applicable you will have to parse yaml config?? + cfg := &config.GlobalConfig // Initialize the metrics server. - metrics, err := metrics.New() + metrics, err := metrics.New(&cfg.Metrics) if err != nil { logger.Error("failed to initialize metrics server", "err", err, diff --git a/go/oasis-node/cmd/node/init.go b/go/oasis-node/cmd/node/init.go index 4870108edd0..c04db5fbeba 100644 --- a/go/oasis-node/cmd/node/init.go +++ b/go/oasis-node/cmd/node/init.go @@ -11,6 +11,8 @@ import ( cmdCommon "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common" "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/background" "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/metrics" + "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/metrics/config" + "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/pprof" cmdSigner "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/signer" ) @@ -81,9 +83,9 @@ func loadOrGenerateIdentity(dataDir string, logger *logging.Logger) (*identity.I } // startMetricServer initializes and starts the metrics reporting server. -func startMetricServer(svcMgr *background.ServiceManager, logger *logging.Logger) (service.BackgroundService, error) { +func startMetricServer(svcMgr *background.ServiceManager, logger *logging.Logger, cfg *config.Config) (service.BackgroundService, error) { // Initialize the metrics server. - metrics, err := metrics.New() + metrics, err := metrics.New(cfg) if err != nil { logger.Error("failed to initialize metrics server", "err", err, diff --git a/go/oasis-node/cmd/node/node.go b/go/oasis-node/cmd/node/node.go index 4cbaa595d1b..7d0e5e1190c 100644 --- a/go/oasis-node/cmd/node/node.go +++ b/go/oasis-node/cmd/node/node.go @@ -27,6 +27,7 @@ import ( "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/background" "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/flags" cmdGrpc "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/grpc" + "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/metrics" "github.com/oasisprotocol/oasis-core/go/p2p" p2pAPI "github.com/oasisprotocol/oasis-core/go/p2p/api" registryAPI "github.com/oasisprotocol/oasis-core/go/registry/api" @@ -151,7 +152,7 @@ func (n *Node) waitReady() { // startRuntimeServices initializes and starts all the services that are required for runtime // support to work. -func (n *Node) startRuntimeServices(genesisDoc *genesisAPI.Document) error { +func (n *Node) startRuntimeServices(genesisDoc *genesisAPI.Document, metricsEnabled bool) error { var err error if n.Sentry, err = sentry.New(n.Consensus, n.Identity); err != nil { return err @@ -169,7 +170,7 @@ func (n *Node) startRuntimeServices(genesisDoc *genesisAPI.Document) error { vaultAPI.RegisterService(grpcSrv, n.Consensus.Vault()) // Initialize runtime workers. - if err = n.initRuntimeWorkers(genesisDoc); err != nil { + if err = n.initRuntimeWorkers(genesisDoc, metricsEnabled); err != nil { n.logger.Error("failed to initialize workers", "err", err, ) @@ -189,11 +190,11 @@ func (n *Node) startRuntimeServices(genesisDoc *genesisAPI.Document) error { return nil } -func (n *Node) initRuntimeWorkers(genesisDoc *genesisAPI.Document) error { +func (n *Node) initRuntimeWorkers(genesisDoc *genesisAPI.Document, metricsEnabled bool) error { var err error // Initialize runtime provisioner. - n.Provisioner, err = provisioner.New(n.dataDir, n.commonStore, n.Identity, n.Consensus, genesisDoc) + n.Provisioner, err = provisioner.New(n.dataDir, n.commonStore, n.Identity, n.Consensus, genesisDoc, metricsEnabled) if err != nil { return err } @@ -217,6 +218,7 @@ func (n *Node) initRuntimeWorkers(genesisDoc *genesisAPI.Document) error { n.Consensus.KeyManager(), n.RuntimeRegistry, n.Provisioner, + metricsEnabled, ) if err != nil { n.logger.Error("failed to initialize common worker", @@ -384,7 +386,7 @@ func (n *Node) startRuntimeWorkers() error { // // Note: the reason for having the named err return value here is for the // deferred func below to propagate the error. -func NewNode() (node *Node, err error) { // nolint: gocyclo +func NewNode(cfg *config.Config) (node *Node, err error) { // nolint: gocyclo logger := cmdCommon.Logger() node = &Node{ @@ -414,7 +416,7 @@ func NewNode() (node *Node, err error) { // nolint: gocyclo // binary is from the logs. logger.Info("Starting oasis-node", "version", version.SoftwareVersion, - "mode", config.GlobalConfig.Mode, + "mode", cfg.Mode, ) if err = verifyElevatedPrivileges(logger); err != nil { @@ -449,7 +451,7 @@ func NewNode() (node *Node, err error) { // nolint: gocyclo crash.LoadViperArgValues() // Initialize and start the metrics reporting server. - if _, err = startMetricServer(node.svcMgr, logger); err != nil { + if _, err = startMetricServer(node.svcMgr, logger, &cfg.Metrics); err != nil { return nil, err } @@ -480,17 +482,23 @@ func NewNode() (node *Node, err error) { // nolint: gocyclo return nil, err } + metricsEnabled := metrics.Enabled(cfg.Metrics.Mode) + // Initialize P2P network. Since libp2p host starts listening immediately when created, make // sure that we don't start it if it is not needed. if genesisDoc.Registry.Parameters.DebugAllowUnroutableAddresses { p2p.DebugForceAllowUnroutableAddresses() } - isArchive := config.GlobalConfig.Mode == config.ModeArchive + isArchive := cfg.Mode == config.ModeArchive if isArchive { node.P2P = p2p.NewNop() } else { - node.P2P, err = p2p.New(node.Identity, node.chainContext, node.commonStore) + var p2pCfg p2p.Config + if err := p2pCfg.Load(&cfg.P2P); err != nil { + return nil, fmt.Errorf("failed to parse p2p config %w", err) + } + node.P2P, err = p2p.New(&p2pCfg, node.Identity, node.chainContext, node.commonStore) if err != nil { return nil, err } @@ -525,7 +533,7 @@ func NewNode() (node *Node, err error) { // nolint: gocyclo // Initialize consensus backend. switch backend := genesisDoc.Consensus.Backend; backend { case cometbftAPI.BackendName: - node.Consensus, err = cometbft.New(node.svcMgr.Ctx, node.dataDir, node.Identity, node.Upgrader, genesis, genesisDoc, node.P2P) + node.Consensus, err = cometbft.New(node.svcMgr.Ctx, node.dataDir, node.Identity, node.Upgrader, genesis, genesisDoc, node.P2P, metricsEnabled) if err != nil { logger.Error("failed to initialize cometbft consensus backend", "err", err, @@ -563,7 +571,7 @@ func NewNode() (node *Node, err error) { // nolint: gocyclo // If the consensus backend supports communicating with consensus services, we can also start // all services required for runtime operation. if node.Consensus.SupportedFeatures().Has(consensusAPI.FeatureServices) { - if err = node.startRuntimeServices(genesisDoc); err != nil { + if err = node.startRuntimeServices(genesisDoc, metricsEnabled); err != nil { logger.Error("failed to initialize runtime services", "err", err, ) diff --git a/go/oasis-node/cmd/node/run.go b/go/oasis-node/cmd/node/run.go index 959461cebb9..76718b71796 100644 --- a/go/oasis-node/cmd/node/run.go +++ b/go/oasis-node/cmd/node/run.go @@ -26,11 +26,13 @@ func Run(_ *cobra.Command, _ []string) { err error ) - switch config.GlobalConfig.Mode { + cfg := &config.GlobalConfig + + switch cfg.Mode { case config.ModeSeed: - node, err = NewSeedNode() + node, err = NewSeedNode(cfg) default: - node, err = NewNode() + node, err = NewNode(cfg) } switch { diff --git a/go/oasis-node/cmd/node/seed.go b/go/oasis-node/cmd/node/seed.go index d3b10eebd96..6a8a17cbcb7 100644 --- a/go/oasis-node/cmd/node/seed.go +++ b/go/oasis-node/cmd/node/seed.go @@ -9,6 +9,7 @@ import ( "github.com/oasisprotocol/oasis-core/go/common/logging" "github.com/oasisprotocol/oasis-core/go/common/persistent" "github.com/oasisprotocol/oasis-core/go/common/version" + "github.com/oasisprotocol/oasis-core/go/config" cmtSeed "github.com/oasisprotocol/oasis-core/go/consensus/cometbft/seed" controlApi "github.com/oasisprotocol/oasis-core/go/control/api" genesisFile "github.com/oasisprotocol/oasis-core/go/genesis/file" @@ -55,7 +56,7 @@ func (n *SeedNode) Cleanup() { } // NewSeedNode initializes the seed node. -func NewSeedNode() (node *SeedNode, err error) { +func NewSeedNode(cfg *config.Config) (node *SeedNode, err error) { logger := cmdCommon.Logger() node = &SeedNode{ @@ -151,7 +152,7 @@ func NewSeedNode() (node *SeedNode, err error) { // Initialize and start the libp2p seed. var seedCfg p2p.SeedConfig - if err = seedCfg.Load(); err != nil { + if err = seedCfg.Load(&cfg.P2P); err != nil { return nil, fmt.Errorf("failed to load libp2p seed config: %w", err) } seedCfg.Signer = node.identity.P2PSigner diff --git a/go/oasis-node/node_test.go b/go/oasis-node/node_test.go index 4b361846c9f..9ea6b429f2a 100644 --- a/go/oasis-node/node_test.go +++ b/go/oasis-node/node_test.go @@ -193,7 +193,8 @@ func newTestNode(t *testing.T) *testNode { start: time.Now(), } t.Logf("starting node, data directory: %v", dataDir) - n.Node, err = node.NewNode() + cfg := &config.GlobalConfig + n.Node, err = node.NewNode(cfg) require.NoError(err, "start node") // Add the testNode to the newly generated entity's list of nodes diff --git a/go/p2p/config.go b/go/p2p/config.go new file mode 100644 index 00000000000..ae64c3d35bd --- /dev/null +++ b/go/p2p/config.go @@ -0,0 +1,244 @@ +package p2p + +import ( + "fmt" + "net" + "time" + + "github.com/libp2p/go-libp2p/core" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" + + "github.com/oasisprotocol/oasis-core/go/common/crypto/signature" + "github.com/oasisprotocol/oasis-core/go/common/node" + "github.com/oasisprotocol/oasis-core/go/common/persistent" + "github.com/oasisprotocol/oasis-core/go/common/version" + "github.com/oasisprotocol/oasis-core/go/p2p/api" + "github.com/oasisprotocol/oasis-core/go/p2p/config" + "github.com/oasisprotocol/oasis-core/go/worker/common/configparser" +) + +// Config describes a set of P2P settings for a peer. +type Config struct { + Addresses []multiaddr.Multiaddr + + HostConfig + GossipSubConfig + BootstrapDiscoveryConfig +} + +// HostConfig describes a set of settings for a host. +type HostConfig struct { + Signer signature.Signer + + UserAgent string + ListenAddr multiaddr.Multiaddr + Port uint16 + + ConnManagerConfig + ConnGaterConfig +} + +// ConnManagerConfig describes a set of settings for a connection manager. +type ConnManagerConfig struct { + MinPeers int + MaxPeers int + GracePeriod time.Duration + PersistentPeers []peer.ID +} + +// ConnGaterConfig describes a set of settings for a connection gater. +type ConnGaterConfig struct { + BlockedPeers []net.IP +} + +// GossipSubConfig describes a set of settings for a gossip pubsub. +type GossipSubConfig struct { + // XXX: Main config has int64, but here just int -- investigate. + PeerOutboundQueueSize int + ValidateQueueSize int + ValidateThrottle int + + PersistentPeers []peer.AddrInfo +} + +// Load loads a default P2P configuration. +func (cfg *Config) Load(yamlCfg *config.Config) error { + rawAddresses, err := configparser.ParseAddressList(yamlCfg.Registration.Addresses) + if err != nil { + return fmt.Errorf("failed to parse address list: %w", err) + } + var addresses []multiaddr.Multiaddr + for _, addr := range rawAddresses { + var mAddr multiaddr.Multiaddr + mAddr, err = manet.FromNetAddr(addr.ToTCPAddr()) + if err != nil { + return fmt.Errorf("failed to convert address to multiaddress: %w", err) + } + addresses = append(addresses, mAddr) + } + + var hostCfg HostConfig + if err := hostCfg.Load(yamlCfg); err != nil { + return fmt.Errorf("failed to load host config: %w", err) + } + + var gossipSubCfg GossipSubConfig + if err := gossipSubCfg.Load(yamlCfg); err != nil { + return fmt.Errorf("failed to load gossipsub config: %w", err) + } + + var bootstrapCfg BootstrapDiscoveryConfig + if err := bootstrapCfg.Load(yamlCfg); err != nil { + return fmt.Errorf("failed to load bootstrap config: %w", err) + } + + cfg.Addresses = addresses + cfg.HostConfig = hostCfg + cfg.GossipSubConfig = gossipSubCfg + cfg.BootstrapDiscoveryConfig = bootstrapCfg + + return nil +} + +// Load loads host configuration. +func (cfg *HostConfig) Load(yamlCfg *config.Config) error { + userAgent := fmt.Sprintf("oasis-core/%s", version.SoftwareVersion) + + port := yamlCfg.Port + + // Listen for connections on all interfaces. + listenAddr, err := multiaddr.NewMultiaddr( + fmt.Sprintf("/ip4/0.0.0.0/tcp/%d", port), + ) + if err != nil { + return fmt.Errorf("failed to create multiaddress: %w", err) + } + + var cmCfg ConnManagerConfig + if err = cmCfg.Load(&yamlCfg.ConnectionManager); err != nil { + return fmt.Errorf("failed to load connection manager config: %w", err) + } + + var cgCfg ConnGaterConfig + if err = cgCfg.Load(yamlCfg.ConnectionGater.BlockedPeerIPs); err != nil { + return fmt.Errorf("failed to load connection gater config: %w", err) + } + + cfg.UserAgent = userAgent + cfg.Port = port + cfg.ListenAddr = listenAddr + cfg.ConnManagerConfig = cmCfg + cfg.ConnGaterConfig = cgCfg + + return nil +} + +// Load loads connection manager configuration. +func (cfg *ConnManagerConfig) Load(yamlCfg *config.ConnectionManagerConfig) error { + persistentPeersMap := make(map[core.PeerID]struct{}) + for _, pp := range yamlCfg.PersistentPeers { + var addr node.ConsensusAddress + if err := addr.UnmarshalText([]byte(pp)); err != nil { + return fmt.Errorf("malformed address (expected pubkey@IP:port): %w", err) + } + + pid, err := api.PublicKeyToPeerID(addr.ID) + if err != nil { + return fmt.Errorf("invalid public key (%s): %w", addr.ID, err) + } + + persistentPeersMap[pid] = struct{}{} + } + + persistentPeers := make([]peer.ID, 0) + for pid := range persistentPeersMap { + persistentPeers = append(persistentPeers, pid) + } + + cfg.MinPeers = yamlCfg.MaxNumPeers + cfg.MaxPeers = cfg.MinPeers + peersHighWatermarkDelta + cfg.GracePeriod = yamlCfg.PeerGracePeriod + cfg.PersistentPeers = persistentPeers + + return nil +} + +// Load loads connection gater configuration. +func (cfg *ConnGaterConfig) Load(blocked []string) error { + blockedPeers := make([]net.IP, 0) + for _, blockedIP := range blocked { + parsedIP := net.ParseIP(blockedIP) + if parsedIP == nil { + return fmt.Errorf("malformed blocked IP: %s", blockedIP) + } + blockedPeers = append(blockedPeers, parsedIP) + } + + cfg.BlockedPeers = blockedPeers + + return nil +} + +// Load loads gossipsub configuration. +func (cfg *GossipSubConfig) Load(yamlCfg *config.Config) error { + persistentPeers, err := api.AddrInfosFromConsensusAddrs(yamlCfg.ConnectionManager.PersistentPeers) + if err != nil { + return fmt.Errorf("failed to convert persistent peers' addresses: %w", err) + } + + cfg.PeerOutboundQueueSize = yamlCfg.Gossipsub.PeerOutboundQueueSize + cfg.ValidateQueueSize = yamlCfg.Gossipsub.ValidateQueueSize + cfg.ValidateThrottle = yamlCfg.Gossipsub.ValidateThrottle + cfg.PersistentPeers = persistentPeers + + return nil +} + +// BootstrapDiscoveryConfig describes a set of settings for a discovery. +type BootstrapDiscoveryConfig struct { + Enable bool + Seeds []peer.AddrInfo + RetentionPeriod time.Duration +} + +// Load loads bootstrap discovery configuration. +func (cfg *BootstrapDiscoveryConfig) Load(yamlCfg *config.Config) error { + seeds, err := api.AddrInfosFromConsensusAddrs(yamlCfg.Seeds) + if err != nil { + return fmt.Errorf("failed to convert seeds' addresses: %w", err) + } + + cfg.Seeds = seeds + cfg.Enable = yamlCfg.Discovery.Bootstrap.Enable + cfg.RetentionPeriod = yamlCfg.Discovery.Bootstrap.RetentionPeriod + + return nil +} + +// SeedConfig describes a set of settings for a seed. +type SeedConfig struct { + CommonStore *persistent.CommonStore + + HostConfig + BootstrapDiscoveryConfig +} + +// Load loads seed configuration. +func (cfg *SeedConfig) Load(yamlCfg *config.Config) error { + var hostCfg HostConfig + if err := hostCfg.Load(yamlCfg); err != nil { + return fmt.Errorf("failed to load host config: %w", err) + } + + var bootstrapCfg BootstrapDiscoveryConfig + if err := bootstrapCfg.Load(yamlCfg); err != nil { + return fmt.Errorf("failed to load bootstrap config: %w", err) + } + + cfg.HostConfig = hostCfg + cfg.BootstrapDiscoveryConfig = bootstrapCfg + + return nil +} diff --git a/go/p2p/host.go b/go/p2p/host.go index 71bdbbaa167..decd7fa96e2 100644 --- a/go/p2p/host.go +++ b/go/p2p/host.go @@ -2,38 +2,18 @@ package p2p import ( "fmt" - "net" - "time" "github.com/libp2p/go-libp2p" - "github.com/libp2p/go-libp2p/core" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/network" - "github.com/libp2p/go-libp2p/core/peer" rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" "github.com/libp2p/go-libp2p/p2p/net/conngater" "github.com/libp2p/go-libp2p/p2p/net/connmgr" - "github.com/multiformats/go-multiaddr" - "github.com/oasisprotocol/oasis-core/go/common/crypto/signature" - "github.com/oasisprotocol/oasis-core/go/common/node" - "github.com/oasisprotocol/oasis-core/go/common/version" "github.com/oasisprotocol/oasis-core/go/config" "github.com/oasisprotocol/oasis-core/go/p2p/api" ) -// HostConfig describes a set of settings for a host. -type HostConfig struct { - Signer signature.Signer - - UserAgent string - ListenAddr multiaddr.Multiaddr - Port uint16 - - ConnManagerConfig - ConnGaterConfig -} - // NewHost constructs a new libp2p host. func NewHost(cfg *HostConfig) (host.Host, *conngater.BasicConnectionGater, error) { id := api.SignerToPrivKey(cfg.Signer) @@ -77,46 +57,6 @@ func (cfg *HostConfig) NewHost() (host.Host, *conngater.BasicConnectionGater, er return NewHost(cfg) } -// Load loads host configuration. -func (cfg *HostConfig) Load() error { - userAgent := fmt.Sprintf("oasis-core/%s", version.SoftwareVersion) - port := config.GlobalConfig.P2P.Port - - // Listen for connections on all interfaces. - listenAddr, err := multiaddr.NewMultiaddr( - fmt.Sprintf("/ip4/0.0.0.0/tcp/%d", port), - ) - if err != nil { - return fmt.Errorf("failed to create multiaddress: %w", err) - } - - var cmCfg ConnManagerConfig - if err = cmCfg.Load(); err != nil { - return fmt.Errorf("failed to load connection manager config: %w", err) - } - - var cgCfg ConnGaterConfig - if err = cgCfg.Load(); err != nil { - return fmt.Errorf("failed to load connection gater config: %w", err) - } - - cfg.UserAgent = userAgent - cfg.Port = port - cfg.ListenAddr = listenAddr - cfg.ConnManagerConfig = cmCfg - cfg.ConnGaterConfig = cgCfg - - return nil -} - -// ConnManagerConfig describes a set of settings for a connection manager. -type ConnManagerConfig struct { - MinPeers int - MaxPeers int - GracePeriod time.Duration - PersistentPeers []peer.ID -} - // NewConnManager constructs a new connection manager. func NewConnManager(cfg *ConnManagerConfig) (*connmgr.BasicConnMgr, error) { gracePeriod := connmgr.WithGracePeriod(cfg.GracePeriod) @@ -135,41 +75,6 @@ func (cfg *ConnManagerConfig) NewConnManager() (*connmgr.BasicConnMgr, error) { return NewConnManager(cfg) } -// Load loads connection manager configuration. -func (cfg *ConnManagerConfig) Load() error { - persistentPeersMap := make(map[core.PeerID]struct{}) - for _, pp := range config.GlobalConfig.P2P.ConnectionManager.PersistentPeers { - var addr node.ConsensusAddress - if err := addr.UnmarshalText([]byte(pp)); err != nil { - return fmt.Errorf("malformed address (expected pubkey@IP:port): %w", err) - } - - pid, err := api.PublicKeyToPeerID(addr.ID) - if err != nil { - return fmt.Errorf("invalid public key (%s): %w", addr.ID, err) - } - - persistentPeersMap[pid] = struct{}{} - } - - persistentPeers := make([]peer.ID, 0) - for pid := range persistentPeersMap { - persistentPeers = append(persistentPeers, pid) - } - - cfg.MinPeers = config.GlobalConfig.P2P.ConnectionManager.MaxNumPeers - cfg.MaxPeers = cfg.MinPeers + peersHighWatermarkDelta - cfg.GracePeriod = config.GlobalConfig.P2P.ConnectionManager.PeerGracePeriod - cfg.PersistentPeers = persistentPeers - - return nil -} - -// ConnGaterConfig describes a set of settings for a connection gater. -type ConnGaterConfig struct { - BlockedPeers []net.IP -} - // NewConnGater constructs a new connection gater. func NewConnGater(cfg *ConnGaterConfig) (*conngater.BasicConnectionGater, error) { // Set up a connection gater and block blacklisted peers. @@ -191,22 +96,6 @@ func (cfg *ConnGaterConfig) NewConnGater() (*conngater.BasicConnectionGater, err return NewConnGater(cfg) } -// Load loads connection gater configuration. -func (cfg *ConnGaterConfig) Load() error { - blockedPeers := make([]net.IP, 0) - for _, blockedIP := range config.GlobalConfig.P2P.ConnectionGater.BlockedPeerIPs { - parsedIP := net.ParseIP(blockedIP) - if parsedIP == nil { - return fmt.Errorf("malformed blocked IP: %s", blockedIP) - } - blockedPeers = append(blockedPeers, parsedIP) - } - - cfg.BlockedPeers = blockedPeers - - return nil -} - // NewResourceManager constructs a new resource manager. func NewResourceManager() (network.ResourceManager, error) { // Use the default resource manager for non-seed nodes. diff --git a/go/p2p/metrics.go b/go/p2p/metrics.go index 9c649f314b0..e5aa5f98264 100644 --- a/go/p2p/metrics.go +++ b/go/p2p/metrics.go @@ -5,8 +5,6 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" - - cmmetrics "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/metrics" ) const metricsUpdateInterval = 60 * time.Second @@ -46,9 +44,6 @@ var ( func (p *p2p) metricsWorker() { defer close(p.metricsClosedCh) - if !cmmetrics.Enabled() { - return - } metricsOnce.Do(func() { prometheus.MustRegister(p2pCollectors...) diff --git a/go/p2p/p2p.go b/go/p2p/p2p.go index c74b243989f..26dac334468 100644 --- a/go/p2p/p2p.go +++ b/go/p2p/p2p.go @@ -26,13 +26,13 @@ import ( "github.com/oasisprotocol/oasis-core/go/common/node" "github.com/oasisprotocol/oasis-core/go/common/persistent" "github.com/oasisprotocol/oasis-core/go/config" + "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/metrics" "github.com/oasisprotocol/oasis-core/go/p2p/api" "github.com/oasisprotocol/oasis-core/go/p2p/discovery/bootstrap" "github.com/oasisprotocol/oasis-core/go/p2p/peermgmt" "github.com/oasisprotocol/oasis-core/go/p2p/protocol" "github.com/oasisprotocol/oasis-core/go/p2p/rpc" registryAPI "github.com/oasisprotocol/oasis-core/go/registry/api" - "github.com/oasisprotocol/oasis-core/go/worker/common/configparser" ) const ( @@ -100,7 +100,10 @@ func (p *p2p) Start() error { // Unfortunately, we cannot start the host as libp2p starts everything on construction. // However, we can start everything else. p.peerMgr.Start() - go p.metricsWorker() + + if metrics.Enabled(config.GlobalConfig.Metrics.Mode) { + go p.metricsWorker() + } return nil } @@ -362,12 +365,7 @@ func messageIdFn(pmsg *pb.Message) string { // nolint: revive } // New creates a new P2P node. -func New(identity *identity.Identity, chainContext string, store *persistent.CommonStore) (api.Service, error) { - var cfg Config - if err := cfg.Load(); err != nil { - return nil, fmt.Errorf("p2p: failed to load peer config: %w", err) - } - +func New(cfg *Config, identity *identity.Identity, chainContext string, store *persistent.CommonStore) (api.Service, error) { // Create the P2P host. cfg.HostConfig.Signer = identity.P2PSigner host, cg, err := NewHost(&cfg.HostConfig) @@ -442,97 +440,3 @@ func New(identity *identity.Identity, chainContext string, store *persistent.Com logger: logger, }, nil } - -// Config describes a set of P2P settings for a peer. -type Config struct { - Addresses []multiaddr.Multiaddr - - HostConfig - GossipSubConfig - BootstrapDiscoveryConfig -} - -// Load loads P2P configuration. -func (cfg *Config) Load() error { - rawAddresses, err := configparser.ParseAddressList(config.GlobalConfig.P2P.Registration.Addresses) - if err != nil { - return fmt.Errorf("failed to parse address list: %w", err) - } - var addresses []multiaddr.Multiaddr - for _, addr := range rawAddresses { - var mAddr multiaddr.Multiaddr - mAddr, err = manet.FromNetAddr(addr.ToTCPAddr()) - if err != nil { - return fmt.Errorf("failed to convert address to multiaddress: %w", err) - } - addresses = append(addresses, mAddr) - } - - var hostCfg HostConfig - if err := hostCfg.Load(); err != nil { - return fmt.Errorf("failed to load host config: %w", err) - } - - var gossipSubCfg GossipSubConfig - if err := gossipSubCfg.Load(); err != nil { - return fmt.Errorf("failed to load gossipsub config: %w", err) - } - - var bootstrapCfg BootstrapDiscoveryConfig - if err := bootstrapCfg.Load(); err != nil { - return fmt.Errorf("failed to load bootstrap config: %w", err) - } - - cfg.Addresses = addresses - cfg.HostConfig = hostCfg - cfg.GossipSubConfig = gossipSubCfg - cfg.BootstrapDiscoveryConfig = bootstrapCfg - - return nil -} - -// GossipSubConfig describes a set of settings for a gossip pubsub. -type GossipSubConfig struct { - // XXX: Main config has int64, but here just int -- investigate. - PeerOutboundQueueSize int - ValidateQueueSize int - ValidateThrottle int - - PersistentPeers []peer.AddrInfo -} - -// Load loads gossipsub configuration. -func (cfg *GossipSubConfig) Load() error { - persistentPeers, err := api.AddrInfosFromConsensusAddrs(config.GlobalConfig.P2P.ConnectionManager.PersistentPeers) - if err != nil { - return fmt.Errorf("failed to convert persistent peers' addresses: %w", err) - } - - cfg.PeerOutboundQueueSize = config.GlobalConfig.P2P.Gossipsub.PeerOutboundQueueSize - cfg.ValidateQueueSize = config.GlobalConfig.P2P.Gossipsub.ValidateQueueSize - cfg.ValidateThrottle = config.GlobalConfig.P2P.Gossipsub.ValidateThrottle - cfg.PersistentPeers = persistentPeers - - return nil -} - -// BootstrapDiscoveryConfig describes a set of settings for a discovery. -type BootstrapDiscoveryConfig struct { - Enable bool - Seeds []peer.AddrInfo - RetentionPeriod time.Duration -} - -// Load loads bootstrap discovery configuration. -func (cfg *BootstrapDiscoveryConfig) Load() error { - seeds, err := api.AddrInfosFromConsensusAddrs(config.GlobalConfig.P2P.Seeds) - if err != nil { - return fmt.Errorf("failed to convert seeds' addresses: %w", err) - } - - cfg.Seeds = seeds - cfg.Enable = config.GlobalConfig.P2P.Discovery.Bootstrap.Enable - cfg.RetentionPeriod = config.GlobalConfig.P2P.Discovery.Bootstrap.RetentionPeriod - - return nil -} diff --git a/go/p2p/seed.go b/go/p2p/seed.go index 515bc40e130..f7b7ff39374 100644 --- a/go/p2p/seed.go +++ b/go/p2p/seed.go @@ -2,7 +2,6 @@ package p2p import ( "context" - "fmt" "sort" "sync" @@ -10,7 +9,6 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/oasisprotocol/oasis-core/go/common/logging" - "github.com/oasisprotocol/oasis-core/go/common/persistent" "github.com/oasisprotocol/oasis-core/go/p2p/api" "github.com/oasisprotocol/oasis-core/go/p2p/backup" "github.com/oasisprotocol/oasis-core/go/p2p/discovery/bootstrap" @@ -148,33 +146,7 @@ func (s *seedNode) registerProtocolServer(srv rpc.Server) { ) } -// SeedConfig describes a set of settings for a seed. -type SeedConfig struct { - CommonStore *persistent.CommonStore - - HostConfig - BootstrapDiscoveryConfig -} - // NewSeed creates a new P2P seed node service. func (cfg *SeedConfig) NewSeed() (api.SeedService, error) { return NewSeedNode(cfg) } - -// Load loads seed configuration. -func (cfg *SeedConfig) Load() error { - var hostCfg HostConfig - if err := hostCfg.Load(); err != nil { - return fmt.Errorf("failed to load host config: %w", err) - } - - var bootstrapCfg BootstrapDiscoveryConfig - if err := bootstrapCfg.Load(); err != nil { - return fmt.Errorf("failed to load bootstrap config: %w", err) - } - - cfg.HostConfig = hostCfg - cfg.BootstrapDiscoveryConfig = bootstrapCfg - - return nil -} diff --git a/go/runtime/host/loadbalance/metrics.go b/go/runtime/host/loadbalance/metrics.go index 64cff313c67..a4804fe0d8c 100644 --- a/go/runtime/host/loadbalance/metrics.go +++ b/go/runtime/host/loadbalance/metrics.go @@ -4,8 +4,6 @@ import ( "sync" "github.com/prometheus/client_golang/prometheus" - - "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/metrics" ) var ( @@ -31,12 +29,8 @@ var ( metricsOnce sync.Once ) -// initMetrics registers the metrics collectors if metrics are enabled. +// initMetrics registers the metrics collectors. func initMetrics() { - if !metrics.Enabled() { - return - } - metricsOnce.Do(func() { prometheus.MustRegister(nodeCollectors...) }) diff --git a/go/runtime/host/loadbalance/provisioner.go b/go/runtime/host/loadbalance/provisioner.go index 480c29e9a48..77b900d8e6d 100644 --- a/go/runtime/host/loadbalance/provisioner.go +++ b/go/runtime/host/loadbalance/provisioner.go @@ -13,13 +13,15 @@ type lbProvisioner struct { } // NewProvisioner creates a load-balancing runtime provisioner. -func NewProvisioner(inner host.Provisioner, numInstances int) host.Provisioner { +func NewProvisioner(inner host.Provisioner, numInstances int, metricsEnabled bool) host.Provisioner { if numInstances < 2 { // If there is only a single instance configured just return the inner provisioner. return inner } - initMetrics() + if metricsEnabled { + initMetrics() + } return &lbProvisioner{ inner: inner, diff --git a/go/runtime/host/protocol/connection.go b/go/runtime/host/protocol/connection.go index 763492db990..4c41b2596e3 100644 --- a/go/runtime/host/protocol/connection.go +++ b/go/runtime/host/protocol/connection.go @@ -15,7 +15,6 @@ import ( "github.com/oasisprotocol/oasis-core/go/common/errors" "github.com/oasisprotocol/oasis-core/go/common/logging" "github.com/oasisprotocol/oasis-core/go/common/version" - "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/metrics" ) const ( @@ -162,6 +161,8 @@ type connection struct { // nolint: maligned quitWg sync.WaitGroup logger *logging.Logger + + metricsEnabled bool } func (c *connection) getState() state { @@ -248,7 +249,7 @@ func (c *connection) Call(ctx context.Context, body *Body) (*Body, error) { func (c *connection) call(ctx context.Context, body *Body) (result *Body, err error) { start := time.Now() defer func() { - if !metrics.Enabled() { + if !c.metricsEnabled { return } @@ -545,8 +546,10 @@ func (c *connection) InitHost(ctx context.Context, conn net.Conn, hi *HostInfo) } // NewConnection creates a new uninitialized RHP connection. -func NewConnection(logger *logging.Logger, runtimeID common.Namespace, handler Handler) (Connection, error) { - initMetrics() +func NewConnection(logger *logging.Logger, runtimeID common.Namespace, handler Handler, metricsEnabled bool) (Connection, error) { + if metricsEnabled { + initMetrics() + } return &connection{ runtimeID: runtimeID, @@ -557,5 +560,6 @@ func NewConnection(logger *logging.Logger, runtimeID common.Namespace, handler H outCh: make(chan *Message), closeCh: make(chan struct{}), logger: logger, + metricsEnabled: metricsEnabled, }, nil } diff --git a/go/runtime/host/protocol/connection_test.go b/go/runtime/host/protocol/connection_test.go index ebe2ab45951..3a0ea6c2ee6 100644 --- a/go/runtime/host/protocol/connection_test.go +++ b/go/runtime/host/protocol/connection_test.go @@ -40,7 +40,7 @@ func TestClose(t *testing.T) { logger := logging.GetLogger("test") handlerA := &testHandler{} - protoA, err := NewConnection(logger, runtimeID, handlerA) + protoA, err := NewConnection(logger, runtimeID, handlerA, false) require.NoError(err, "NewConnection") require.NotPanics(func() { protoA.Close() }) } @@ -52,10 +52,10 @@ func TestEchoRequestResponse(t *testing.T) { logger := logging.GetLogger("test") connA, connB := net.Pipe() handlerA := &testHandler{} - protoA, err := NewConnection(logger, runtimeID, handlerA) + protoA, err := NewConnection(logger, runtimeID, handlerA, false) require.NoError(err, "A.New()") handlerB := &testHandler{} - protoB, err := NewConnection(logger, runtimeID, handlerB) + protoB, err := NewConnection(logger, runtimeID, handlerB, false) require.NoError(err, "B.New()") err = protoA.InitGuest(connA) @@ -109,10 +109,10 @@ func TestBigMessage(t *testing.T) { connA, connB := net.Pipe() handlerA := &testHandler{} - protoA, err := NewConnection(logger, runtimeID, handlerA) + protoA, err := NewConnection(logger, runtimeID, handlerA, false) require.NoError(err, "A.New()") handlerB := &testHandler{} - protoB, err := NewConnection(logger, runtimeID, handlerB) + protoB, err := NewConnection(logger, runtimeID, handlerB, false) require.NoError(err, "B.New()") err = protoA.InitGuest(connA) diff --git a/go/runtime/host/protocol/metrics.go b/go/runtime/host/protocol/metrics.go index 905ba7b3e04..b4a8e95e92e 100644 --- a/go/runtime/host/protocol/metrics.go +++ b/go/runtime/host/protocol/metrics.go @@ -4,8 +4,6 @@ import ( "sync" "github.com/prometheus/client_golang/prometheus" - - "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/metrics" ) var ( @@ -49,10 +47,6 @@ var ( // initMetrics registers the metrics collectors. func initMetrics() { - if !metrics.Enabled() { - return - } - metricsOnce.Do(func() { prometheus.MustRegister(rhpCollectors...) }) diff --git a/go/runtime/host/provisioner/provisioner.go b/go/runtime/host/provisioner/provisioner.go index fcaff1464ae..6c9fcd6ba22 100644 --- a/go/runtime/host/provisioner/provisioner.go +++ b/go/runtime/host/provisioner/provisioner.go @@ -36,6 +36,7 @@ func New( identity *identity.Identity, consensus consensus.Service, genesisDoc *genesisAPI.Document, + metricsEnabled bool, ) (runtimeHost.Provisioner, error) { // Initialize the IAS proxy client. ias, err := ias.New(identity) @@ -56,7 +57,7 @@ func New( } // Create runtime provisioner. - return createProvisioner(dataDir, commonStore, identity, consensus, hostInfo, ias, qs) + return createProvisioner(dataDir, commonStore, identity, consensus, hostInfo, ias, qs, metricsEnabled) } func createHostInfo(genesisDoc *genesisAPI.Document) (*hostProtocol.HostInfo, error) { @@ -88,6 +89,7 @@ func createProvisioner( hostInfo *hostProtocol.HostInfo, ias []iasAPI.Endpoint, qs pcs.QuoteService, + metricsEnabled bool, ) (runtimeHost.Provisioner, error) { var err error var insecureNoSandbox bool @@ -133,6 +135,7 @@ func createProvisioner( HostInfo: hostInfo, InsecureNoSandbox: insecureNoSandbox, SandboxBinaryPath: sandboxBinary, + MetricsEnabled: metricsEnabled, }) if err != nil { return nil, fmt.Errorf("failed to create runtime provisioner: %w", err) @@ -160,6 +163,7 @@ func createProvisioner( InsecureNoSandbox: insecureNoSandbox, InsecureMock: insecureMock, RuntimeAttestInterval: attestInterval, + MetricsEnabled: metricsEnabled, }) if err != nil { return nil, fmt.Errorf("failed to create SGX runtime provisioner: %w", err) @@ -187,6 +191,7 @@ func createProvisioner( Identity: identity, CidPool: cidPool, RuntimeAttestInterval: attestInterval, + MetricsEnabled: metricsEnabled, }) if err != nil { return nil, fmt.Errorf("failed to create TDX runtime provisioner: %w", err) @@ -195,7 +200,7 @@ func createProvisioner( // Configure optional load balancing. for tee, rp := range provisioners { numInstances := int(config.GlobalConfig.Runtime.LoadBalancer.NumInstances) - provisioners[tee] = hostLoadBalance.NewProvisioner(rp, numInstances) + provisioners[tee] = hostLoadBalance.NewProvisioner(rp, numInstances, metricsEnabled) } // Create a composite provisioner to provision the individual components. diff --git a/go/runtime/host/sandbox/host.go b/go/runtime/host/sandbox/host.go index 01fd68a7cd7..7b5df05f345 100644 --- a/go/runtime/host/sandbox/host.go +++ b/go/runtime/host/sandbox/host.go @@ -276,7 +276,7 @@ func (h *sandboxHost) startProcess(ctx context.Context) (err error) { "pid", p.GetPID(), ) - pc, err := protocol.NewConnection(h.logger, h.id, h.rtCfg.MessageHandler) + pc, err := protocol.NewConnection(h.logger, h.id, h.rtCfg.MessageHandler, h.cfg.MetricsEnabled) if err != nil { return fmt.Errorf("failed to create connection: %w", err) } diff --git a/go/runtime/host/sandbox/provisioner.go b/go/runtime/host/sandbox/provisioner.go index bcb8ea1f783..e42b60e3739 100644 --- a/go/runtime/host/sandbox/provisioner.go +++ b/go/runtime/host/sandbox/provisioner.go @@ -48,6 +48,9 @@ type Config struct { // InsecureNoSandbox disables the sandbox and runs the runtime binary directly. InsecureNoSandbox bool + + // MetricsEnabled is true if prometheus metrics are enabled. + MetricsEnabled bool } type sandboxProvisioner struct { diff --git a/go/runtime/host/sgx/common/metrics.go b/go/runtime/host/sgx/common/metrics.go index 9dd57b90fac..fedd0c7dced 100644 --- a/go/runtime/host/sgx/common/metrics.go +++ b/go/runtime/host/sgx/common/metrics.go @@ -6,7 +6,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/oasisprotocol/oasis-core/go/common" - "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/metrics" "github.com/oasisprotocol/oasis-core/go/runtime/bundle/component" ) @@ -47,12 +46,8 @@ var ( metricsOnce sync.Once ) -// UpdateAttestationMetrics updates the attestation metrics if metrics are enabled. +// UpdateAttestationMetrics updates the attestation metrics. func UpdateAttestationMetrics(runtimeID common.Namespace, kind component.TEEKind, err error) { - if !metrics.Enabled() { - return - } - runtime := runtimeID.String() kindStr := kind.String() @@ -64,12 +59,8 @@ func UpdateAttestationMetrics(runtimeID common.Namespace, kind component.TEEKind } } -// InitMetrics registers the metrics collectors if metrics are enabled. +// InitMetrics registers the metrics collectors. func InitMetrics() { - if !metrics.Enabled() { - return - } - metricsOnce.Do(func() { prometheus.MustRegister(teeCollectors...) }) diff --git a/go/runtime/host/sgx/provisioner.go b/go/runtime/host/sgx/provisioner.go index 4e20025bc22..6fb476f8fb4 100644 --- a/go/runtime/host/sgx/provisioner.go +++ b/go/runtime/host/sgx/provisioner.go @@ -82,6 +82,9 @@ type Config struct { // This is useful in tests so most SGX code can be tested even on machines that lack SGX. Note // that this also requires quote verification to be skipped. InsecureMock bool + + // MetricsEnabled is true if prometheus metrics are enabled. + MetricsEnabled bool } type sgxProvisioner struct { @@ -107,7 +110,9 @@ func NewProvisioner(cfg Config) (host.Provisioner, error) { cfg.RuntimeAttestInterval = defaultRuntimeAttestInterval } - sgxCommon.InitMetrics() + if cfg.MetricsEnabled { + sgxCommon.InitMetrics() + } p := &sgxProvisioner{ cfg: cfg, @@ -125,6 +130,7 @@ func NewProvisioner(cfg Config) (host.Provisioner, error) { HostInitializer: p.hostInitializer, InsecureNoSandbox: cfg.InsecureNoSandbox, Logger: p.logger, + MetricsEnabled: cfg.MetricsEnabled, }) if err != nil { return nil, err @@ -340,7 +346,7 @@ func (p *sgxProvisioner) initCapabilityTEE(ctx context.Context, cfg *host.Config insecureMock: p.cfg.InsecureMock, } - targetInfo, err := ts.init(ctx, p) + targetInfo, err := ts.init(ctx, p, p.cfg.MetricsEnabled) if err != nil { return nil, fmt.Errorf("error while initializing TEE state: %w", err) } diff --git a/go/runtime/host/sgx/state.go b/go/runtime/host/sgx/state.go index 8d50beca50b..09fe8dc92e3 100644 --- a/go/runtime/host/sgx/state.go +++ b/go/runtime/host/sgx/state.go @@ -23,12 +23,15 @@ type teeState struct { insecureMock bool impl teeStateImpl + + metricsEnabled bool } -func (ts *teeState) init(ctx context.Context, sp *sgxProvisioner) ([]byte, error) { +func (ts *teeState) init(ctx context.Context, sp *sgxProvisioner, metricsEnabled bool) ([]byte, error) { if ts.impl != nil { return nil, fmt.Errorf("already initialized") } + ts.metricsEnabled = metricsEnabled // TODO are you sure?! var ( targetInfo []byte @@ -74,7 +77,9 @@ func (ts *teeState) update(ctx context.Context, sp *sgxProvisioner, conn protoco attestation, err := ts.impl.Update(ctx, sp, conn, report, nonce) - common.UpdateAttestationMetrics(ts.cfg.ID, component.TEEKindSGX, err) + if ts.metricsEnabled { + common.UpdateAttestationMetrics(ts.cfg.ID, component.TEEKindSGX, err) + } return attestation, err } diff --git a/go/runtime/host/tdx/qemu.go b/go/runtime/host/tdx/qemu.go index 07c5eb6afb0..7ef7fbbb6ba 100644 --- a/go/runtime/host/tdx/qemu.go +++ b/go/runtime/host/tdx/qemu.go @@ -67,6 +67,9 @@ type QemuConfig struct { // RuntimeAttestInterval is the interval for periodic runtime re-attestation. If not specified // a default will be used. RuntimeAttestInterval time.Duration + + // MetricsEnabled is true if prometheus metris are enabled. + MetricsEnabled bool } // QemuExtraConfig is the per-runtime QEMU-specific extra configuration. @@ -94,7 +97,9 @@ func NewQemuProvisioner(cfg QemuConfig) (host.Provisioner, error) { cfg.RuntimeAttestInterval = defaultRuntimeAttestInterval } - sgxCommon.InitMetrics() + if cfg.MetricsEnabled { + sgxCommon.InitMetrics() + } p := &qemuProvisioner{ cfg: cfg, @@ -112,6 +117,7 @@ func NewQemuProvisioner(cfg QemuConfig) (host.Provisioner, error) { HostInitializer: p.hostInitializer, InsecureNoSandbox: true, // No sandbox is needed for TDX. Logger: p.logger, + MetricsEnabled: cfg.MetricsEnabled, }) if err != nil { return nil, err @@ -415,7 +421,9 @@ func (p *qemuProvisioner) createPersistentOverlayImage(image string, format stri func (p *qemuProvisioner) updateCapabilityTEE(ctx context.Context, hp *sandbox.HostInitializerParams) (capTEE *node.CapabilityTEE, aerr error) { defer func() { - sgxCommon.UpdateAttestationMetrics(hp.Runtime.ID(), component.TEEKindTDX, aerr) + if p.cfg.MetricsEnabled { + sgxCommon.UpdateAttestationMetrics(hp.Runtime.ID(), component.TEEKindTDX, aerr) + } }() // Issue the RAK report request which will return the full quote in TDX since the attestation diff --git a/go/worker/common/committee/node.go b/go/worker/common/committee/node.go index 6701b34cb70..d65de68ed44 100644 --- a/go/worker/common/committee/node.go +++ b/go/worker/common/committee/node.go @@ -19,7 +19,6 @@ import ( consensus "github.com/oasisprotocol/oasis-core/go/consensus/api" control "github.com/oasisprotocol/oasis-core/go/control/api" keymanager "github.com/oasisprotocol/oasis-core/go/keymanager/api" - cmmetrics "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/metrics" p2pAPI "github.com/oasisprotocol/oasis-core/go/p2p/api" p2pProtocol "github.com/oasisprotocol/oasis-core/go/p2p/protocol" registry "github.com/oasisprotocol/oasis-core/go/registry/api" @@ -192,7 +191,8 @@ type Node struct { CurrentDescriptor *registry.Runtime CurrentEpoch beacon.EpochTime - logger *logging.Logger + logger *logging.Logger + metricsEnabled bool } func (n *Node) getStatusStateLocked() api.StatusState { @@ -234,7 +234,7 @@ func (n *Node) Start() error { } go n.worker() - if cmmetrics.Enabled() { + if n.metricsEnabled { go n.metricsWorker() } @@ -889,6 +889,7 @@ func NewNode( lightProvider consensus.LightProvider, p2pHost p2pAPI.Service, txPoolCfg tpConfig.Config, + metricsEnabled bool, ) (*Node, error) { metricsOnce.Do(func() { prometheus.MustRegister(nodeCollectors...) @@ -923,6 +924,7 @@ func NewNode( quitCh: make(chan struct{}), initCh: make(chan struct{}), logger: logging.GetLogger("worker/common/committee").With("runtime_id", runtime.ID()), + metricsEnabled: metricsEnabled, } // Prepare the key manager client wrapper. diff --git a/go/worker/common/config.go b/go/worker/common/config.go index 79b2243e09a..add8da961c5 100644 --- a/go/worker/common/config.go +++ b/go/worker/common/config.go @@ -15,11 +15,13 @@ type Config struct { // nolint: maligned TxPool tpConfig.Config + MetricsEnabled bool + logger *logging.Logger } // NewConfig creates a new worker config. -func NewConfig() (*Config, error) { +func NewConfig(metricsEnabled bool) (*Config, error) { // Parse sentry configuration. var sentryAddresses []node.TLSAddress for _, v := range config.GlobalConfig.Runtime.SentryAddresses { @@ -33,6 +35,7 @@ func NewConfig() (*Config, error) { cfg := Config{ SentryAddresses: sentryAddresses, TxPool: config.GlobalConfig.Runtime.TxPool, + MetricsEnabled: metricsEnabled, logger: logging.GetLogger("worker/config"), } diff --git a/go/worker/common/worker.go b/go/worker/common/worker.go index fb0b8b3f2ec..b02b06e2d69 100644 --- a/go/worker/common/worker.go +++ b/go/worker/common/worker.go @@ -166,6 +166,7 @@ func (w *Worker) registerRuntime(runtime runtimeRegistry.Runtime) error { w.LightProvider, w.P2P, w.cfg.TxPool, + w.cfg.MetricsEnabled, ) if err != nil { return err @@ -191,6 +192,7 @@ func New( keyManager keymanagerApi.Backend, runtimeRegistry runtimeRegistry.Registry, provisioner host.Provisioner, + metricsEnabled bool, ) (*Worker, error) { var enabled bool switch config.GlobalConfig.Mode { @@ -203,7 +205,7 @@ func New( enabled = true } - cfg, err := NewConfig() + cfg, err := NewConfig(metricsEnabled) if err != nil { return nil, fmt.Errorf("worker/common: failed to initialize config: %w", err) } diff --git a/go/worker/compute/executor/committee/metrics.go b/go/worker/compute/executor/committee/metrics.go index 22c392beaa2..ecc3fdcd055 100644 --- a/go/worker/compute/executor/committee/metrics.go +++ b/go/worker/compute/executor/committee/metrics.go @@ -4,8 +4,6 @@ import ( "sync" "github.com/prometheus/client_golang/prometheus" - - "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/metrics" ) var ( @@ -77,12 +75,8 @@ func (n *Node) getMetricLabels() prometheus.Labels { } } -// initMetrics registers the metrics collectors if metrics are enabled. +// initMetrics registers the metrics collectors. func initMetrics() { - if !metrics.Enabled() { - return - } - metricsOnce.Do(func() { prometheus.MustRegister(nodeCollectors...) }) diff --git a/go/worker/compute/executor/committee/node.go b/go/worker/compute/executor/committee/node.go index d07cba87e7c..107c3721340 100644 --- a/go/worker/compute/executor/committee/node.go +++ b/go/worker/compute/executor/committee/node.go @@ -1545,7 +1545,9 @@ func NewNode( commonCfg commonWorker.Config, roleProvider registration.RoleProvider, ) (*Node, error) { - initMetrics() + if commonCfg.MetricsEnabled { + initMetrics() + } committeeTopic := p2pProtocol.NewTopicKindCommitteeID(commonNode.ChainContext, commonNode.Runtime.ID()) diff --git a/go/worker/registration/worker.go b/go/worker/registration/worker.go index 7a776446849..320847832d7 100644 --- a/go/worker/registration/worker.go +++ b/go/worker/registration/worker.go @@ -28,7 +28,6 @@ import ( consensus "github.com/oasisprotocol/oasis-core/go/consensus/api" control "github.com/oasisprotocol/oasis-core/go/control/api" "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/flags" - cmmetrics "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/metrics" p2p "github.com/oasisprotocol/oasis-core/go/p2p/api" registry "github.com/oasisprotocol/oasis-core/go/registry/api" runtimeRegistry "github.com/oasisprotocol/oasis-core/go/runtime/registry" @@ -1143,7 +1142,7 @@ func (w *Worker) Start() error { } go w.doNodeRegistration() - if cmmetrics.Enabled() { + if w.workerCommonCfg.MetricsEnabled { go w.metricsWorker() }