op-challenger: Add subcommand to continuously run trace providers (et…

…hereum-optimism#11294)

* op-challenger: Add subcommand to continuously run trace providers

* op-challenger: Remove duplicate invalid recording.

op-challenger/cmd/main.go

@@ -56,6 +56,7 @@ func run(ctx context.Context, args []string, action ConfiguredLifecycle) error {
 		MoveCommand,
 		ResolveCommand,
 		ResolveClaimCommand,
+		RunTraceCommand,
 	}
 	app.Action = cliapp.LifecycleCmd(func(ctx *cli.Context, close context.CancelCauseFunc) (cliapp.Lifecycle, error) {
 		logger, err := setupLogging(ctx)

op-challenger/cmd/run_trace.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"context"
+
+	"github.com/ethereum-optimism/optimism/op-challenger/flags"
+	"github.com/ethereum-optimism/optimism/op-challenger/runner"
+	"github.com/ethereum-optimism/optimism/op-service/cliapp"
+	"github.com/urfave/cli/v2"
+)
+
+func RunTrace(ctx *cli.Context, _ context.CancelCauseFunc) (cliapp.Lifecycle, error) {
+
+	logger, err := setupLogging(ctx)
+	if err != nil {
+		return nil, err
+	}
+	logger.Info("Starting trace runner", "version", VersionWithMeta)
+
+	cfg, err := flags.NewConfigFromCLI(ctx, logger)
+	if err != nil {
+		return nil, err
+	}
+	if err := cfg.Check(); err != nil {
+		return nil, err
+	}
+	return runner.NewRunner(logger, cfg), nil
+}
+
+func runTraceFlags() []cli.Flag {
+	return flags.Flags
+}
+
+var RunTraceCommand = &cli.Command{
+	Name:        "run-trace",
+	Usage:       "Continuously runs the specified trace providers in a regular loop",
+	Description: "Runs trace providers against real chain data to confirm compatibility",
+	Action:      cliapp.LifecycleCmd(RunTrace),
+	Flags:       runTraceFlags(),
+}

op-challenger/game/fault/register.go

@@ -204,12 +204,7 @@ func registerAsterisc(
 	selective bool,
 	claimants []common.Address,
 ) error {
-	var prestateSource PrestateSource
-	if cfg.AsteriscAbsolutePreStateBaseURL != nil {
-		prestateSource = prestates.NewMultiPrestateProvider(cfg.AsteriscAbsolutePreStateBaseURL, filepath.Join(cfg.Datadir, "asterisc-prestates"))
-	} else {
-		prestateSource = prestates.NewSinglePrestateSource(cfg.AsteriscAbsolutePreState)
-	}
+	prestateSource := prestates.NewPrestateSource(cfg.AsteriscAbsolutePreStateBaseURL, cfg.AsteriscAbsolutePreState, filepath.Join(cfg.Datadir, "asterisc-prestates"))
 	prestateProviderCache := prestates.NewPrestateProviderCache(m, fmt.Sprintf("prestates-%v", gameType), func(prestateHash common.Hash) (faultTypes.PrestateProvider, error) {
 		prestatePath, err := prestateSource.PrestatePath(prestateHash)
 		if err != nil {
@@ -297,12 +292,7 @@ func registerCannon(
 	selective bool,
 	claimants []common.Address,
 ) error {
-	var prestateSource PrestateSource
-	if cfg.CannonAbsolutePreStateBaseURL != nil {
-		prestateSource = prestates.NewMultiPrestateProvider(cfg.CannonAbsolutePreStateBaseURL, filepath.Join(cfg.Datadir, "cannon-prestates"))
-	} else {
-		prestateSource = prestates.NewSinglePrestateSource(cfg.CannonAbsolutePreState)
-	}
+	prestateSource := prestates.NewPrestateSource(cfg.CannonAbsolutePreStateBaseURL, cfg.CannonAbsolutePreState, filepath.Join(cfg.Datadir, "cannon-prestates"))
 	prestateProviderCache := prestates.NewPrestateProviderCache(m, fmt.Sprintf("prestates-%v", gameType), func(prestateHash common.Hash) (faultTypes.PrestateProvider, error) {
 		prestatePath, err := prestateSource.PrestatePath(prestateHash)
 		if err != nil {

op-challenger/game/fault/trace/prestates/source.go

@@ -0,0 +1,11 @@
+package prestates
+
+import "net/url"
+
+func NewPrestateSource(baseURL *url.URL, path string, localDir string) PrestateSource {
+	if baseURL != nil {
+		return NewMultiPrestateProvider(baseURL, localDir)
+	} else {
+		return NewSinglePrestateSource(path)
+	}
+}

op-challenger/runner/factory.go

@@ -0,0 +1,42 @@
+package runner
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/ethereum-optimism/optimism/op-challenger/config"
+	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/asterisc"
+	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/cannon"
+	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/prestates"
+	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/utils"
+	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/vm"
+	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/types"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+func createTraceProvider(
+	logger log.Logger,
+	m vm.Metricer,
+	cfg *config.Config,
+	prestateSource prestates.PrestateSource,
+	prestateHash common.Hash,
+	traceType types.TraceType,
+	localInputs utils.LocalGameInputs,
+	dir string,
+) (types.TraceProvider, error) {
+	prestate, err := prestateSource.PrestatePath(prestateHash)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get prestate %v: %w", prestateHash, err)
+	}
+
+	switch traceType {
+	case types.TraceTypeCannon:
+		prestateProvider := cannon.NewPrestateProvider(prestate)
+		return cannon.NewTraceProvider(logger, m, cfg.Cannon, prestateProvider, prestate, localInputs, dir, 42), nil
+	case types.TraceTypeAsterisc:
+		prestateProvider := asterisc.NewPrestateProvider(prestate)
+		return asterisc.NewTraceProvider(logger, m, cfg.Asterisc, prestateProvider, prestate, localInputs, dir, 42), nil
+	}
+	return nil, errors.New("invalid trace type")
+}

op-challenger/runner/metrics.go

@@ -0,0 +1,86 @@
+package runner
+
+import (
+	"time"
+
+	contractMetrics "github.com/ethereum-optimism/optimism/op-challenger/game/fault/contracts/metrics"
+	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/types"
+	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const Namespace = "op_challenger_runner"
+
+type Metrics struct {
+	ns       string
+	registry *prometheus.Registry
+	factory  opmetrics.Factory
+	*contractMetrics.ContractMetrics
+
+	vmExecutionTime *prometheus.HistogramVec
+	successTotal    *prometheus.CounterVec
+	failuresTotal   *prometheus.CounterVec
+	invalidTotal    *prometheus.CounterVec
+}
+
+var _ Metricer = (*Metrics)(nil)
+
+// Metrics implementation must implement RegistryMetricer to allow the metrics server to work.
+var _ opmetrics.RegistryMetricer = (*Metrics)(nil)
+
+func NewMetrics() *Metrics {
+	registry := opmetrics.NewRegistry()
+	factory := opmetrics.With(registry)
+
+	return &Metrics{
+		ns:       Namespace,
+		registry: registry,
+		factory:  factory,
+
+		ContractMetrics: contractMetrics.MakeContractMetrics(Namespace, factory),
+
+		vmExecutionTime: factory.NewHistogramVec(prometheus.HistogramOpts{
+			Namespace: Namespace,
+			Name:      "vm_execution_time",
+			Help:      "Time (in seconds) to execute the fault proof VM",
+			Buckets: append(
+				[]float64{1.0, 10.0},
+				prometheus.ExponentialBuckets(30.0, 2.0, 14)...),
+		}, []string{"vm"}),
+		successTotal: factory.NewCounterVec(prometheus.CounterOpts{
+			Namespace: Namespace,
+			Name:      "success_total",
+			Help:      "Number of VM executions that successfully verified the output root",
+		}, []string{"type"}),
+		failuresTotal: factory.NewCounterVec(prometheus.CounterOpts{
+			Namespace: Namespace,
+			Name:      "failures_total",
+			Help:      "Number of failures to execute a VM",
+		}, []string{"type"}),
+		invalidTotal: factory.NewCounterVec(prometheus.CounterOpts{
+			Namespace: Namespace,
+			Name:      "invalid_total",
+			Help:      "Number of runs that determined the output root was invalid",
+		}, []string{"type"}),
+	}
+}
+
+func (m *Metrics) Registry() *prometheus.Registry {
+	return m.registry
+}
+
+func (m *Metrics) RecordVmExecutionTime(vmType string, dur time.Duration) {
+	m.vmExecutionTime.WithLabelValues(vmType).Observe(dur.Seconds())
+}
+
+func (m *Metrics) RecordSuccess(vmType types.TraceType) {
+	m.successTotal.WithLabelValues(vmType.String()).Inc()
+}
+
+func (m *Metrics) RecordFailure(vmType types.TraceType) {
+	m.failuresTotal.WithLabelValues(vmType.String()).Inc()
+}
+
+func (m *Metrics) RecordInvalid(vmType types.TraceType) {
+	m.invalidTotal.WithLabelValues(vmType.String()).Inc()
+}