Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -178,15 +178,15 @@
APPENDIX: How to apply the Apache License to your work.

To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.

Copyright {yyyy} {name of copyright owner}
Copyright [yyyy] [name of copyright owner]

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
Expand All @@ -199,3 +199,33 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

--------------------------------------------------------------------------------

Copyright 2009 The Go Authors.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google LLC nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
19 changes: 0 additions & 19 deletions LICENSES/go.uber.org/multierr/LICENSE.txt

This file was deleted.

19 changes: 0 additions & 19 deletions LICENSES/go.uber.org/zap/LICENSE

This file was deleted.

49 changes: 47 additions & 2 deletions cmd/atecontroller/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,20 +14,26 @@
package main

import (
"context"
"log/slog"
"os"

"github.com/agent-substrate/substrate/cmd/atecontroller/internal/controllers"
"github.com/agent-substrate/substrate/internal/ateapiauth"
"github.com/agent-substrate/substrate/internal/serverboot"
clientv1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
"github.com/go-logr/logr"
"github.com/spf13/pflag"
prombridge "go.opentelemetry.io/contrib/bridges/prometheus"
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
"google.golang.org/grpc"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"

// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
_ "k8s.io/client-go/plugin/pkg/client/auth"
Expand All @@ -39,6 +45,8 @@ var (

ateAPIConnSpec = pflag.String("ateapi-conn-spec", "dns:///api.ate-system.svc:443", "")

logLevelFlag = pflag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.")

otelEndpoint = pflag.String("otel-exporter-otlp-endpoint", os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"),
"OTLP endpoint set on ateom worker pods so they push telemetry. Defaults to the controller's own OTEL_EXPORTER_OTLP_ENDPOINT.")

Expand Down Expand Up @@ -66,9 +74,45 @@ func init() {
utilruntime.Must(clientv1alpha1.AddToScheme(scheme)) // Register our CRD
}

const serviceName = "atecontroller"

// logr verbosity V(n) maps to slog level -n, so V(1) stays below Info until
// --log-level=debug. logr carries no context, so these records have no trace IDs.
func newControllerRuntimeLogger(h slog.Handler) logr.Logger {
return logr.FromSlogHandler(h)
}

func main() {
pflag.Parse()
ctrl.SetLogger(zap.New(zap.UseDevMode(true)))

ctx := context.Background()
serverboot.InitLogger()
if err := serverboot.SetLogLevel(*logLevelFlag); err != nil {
serverboot.Fatal(ctx, "Invalid --log-level", err)
}
ctrl.SetLogger(newControllerRuntimeLogger(slog.Default().Handler()))

// Both providers must be registered before the ateapi client below:
// otelgrpc.NewClientHandler captures the global tracer and meter providers at
// construction, so a later init leaves it bound to the no-op ones.
tp, err := serverboot.InitTracing(ctx, serverboot.TracingOptions{
ServiceName: serviceName,
Sampling: serverboot.ResolveTraceSampling(ctx, serverboot.ParentRatioSampling(serverboot.ControlPlaneTraceRatio)),
})
if err != nil {
serverboot.Fatal(ctx, "Failed to initialize tracing", err)
}
defer serverboot.ShutdownProvider("TracerProvider", tp.Shutdown)

// controller-runtime records reconcile, workqueue, and runtime metrics into its
// own Prometheus registry, which the manager serves on a port nothing scrapes.
// Bridging it as a Producer puts them on the OTLP path instead.
mp, err := serverboot.InitMetricsPushOnly(ctx, serviceName,
prombridge.NewMetricProducer(prombridge.WithGatherer(ctrlmetrics.Registry)))
if err != nil {
serverboot.Fatal(ctx, "Failed to initialize metrics", err)
}
defer serverboot.ShutdownProvider("MeterProvider", mp.Shutdown)

dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{
UseTokenAuth: *ateapiTokenAuth,
Expand All @@ -81,6 +125,7 @@ func main() {
setupLog.Error(err, "building ateapi dial options")
os.Exit(1)
}
dialOpts = append(dialOpts, grpc.WithStatsHandler(otelgrpc.NewClientHandler()))

ateapiConn, err := grpc.NewClient(*ateAPIConnSpec, dialOpts...)
if err != nil {
Expand Down
80 changes: 80 additions & 0 deletions cmd/atecontroller/main_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main

import (
"context"
"log/slog"
"slices"
"strings"
"testing"

prombridge "go.opentelemetry.io/contrib/bridges/prometheus"
ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
)

func TestNewControllerRuntimeLoggerVerbosity(t *testing.T) {
const probe = "verbosity-probe"

tests := []struct {
name string
level slog.Level
verbosity int
wantLog bool
}{
{name: "info keeps V(0)", level: slog.LevelInfo, verbosity: 0, wantLog: true},
{name: "info drops V(1)", level: slog.LevelInfo, verbosity: 1, wantLog: false},
{name: "debug keeps V(1)", level: slog.LevelDebug, verbosity: 1, wantLog: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
var buf strings.Builder
log := newControllerRuntimeLogger(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: tt.level}))

log.V(tt.verbosity).Info(probe)

if got := strings.Contains(buf.String(), probe); got != tt.wantLog {
t.Errorf("logged = %v, want %v (output %q)", got, tt.wantLog, buf.String())
}
})
}
}

// The e2e read-back asserts atecontroller reaches the collector, which holds only
// because the bridged registry already has series before any manager starts: the
// controller_runtime_* vectors are empty until controllers register, so the Go and
// process collectors are what make the first push non-empty.
func TestBridgedRegistryProducesBeforeManagerStart(t *testing.T) {
t.Parallel()

produced, err := prombridge.NewMetricProducer(prombridge.WithGatherer(ctrlmetrics.Registry)).
Produce(context.Background())
if err != nil {
t.Fatalf("Produce: %v", err)
}

var names []string
for _, sm := range produced {
for _, m := range sm.Metrics {
names = append(names, m.Name)
}
}
if len(names) == 0 {
t.Fatal("bridging controller-runtime's registry produced no metrics")
}
if !slices.ContainsFunc(names, func(n string) bool { return strings.HasPrefix(n, "go_") }) {
t.Errorf("no go_* family in %v", names)
}
}
8 changes: 8 additions & 0 deletions docs/observability.md
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,14 @@ For `atenet.router.route.duration`:

The `ate.*` control-plane metric labels are either fixed value sets (operation, outcome, state, class, kind) or scoped to the deployment catalog (template and pool names are operator-created, never derived from request payloads), and the label set varies per operation: resume carries the most dimensions, delete only the operation and error type. `ate.sandbox.class` is derived from the template (each template has exactly one class), so it adds no extra series next to the template labels; it exists so dashboards can aggregate by class without enumerating template names. High-cardinality actor identity (name/uid/atespace) stays off metrics entirely and lives on logs and traces instead.

### Bridged controller-runtime metrics (atecontroller)

atecontroller bridges controller-runtime's private Prometheus registry, which the manager serves on an unscraped `:8080`, onto its OTLP reader. So `controller_runtime_*`, `workqueue_*`, `rest_client_*`, `leader_election_*`, `go_*`, and `process_*` reach the collector too, keeping their Prometheus names because they are upstream instruments and renaming them would break existing controller-runtime dashboards.
Comment thread
JeffLuoo marked this conversation as resolved.

These can be used to answer whether the controller is keeping up, e.g. rising `workqueue_depth` or `workqueue_queue_duration_seconds` means reconciles are falling behind, and `controller_runtime_reconcile_errors_total` says which controller.

Note that controller-runtime enables native histograms on `controller_runtime_reconcile_time_seconds`, `workqueue_queue_duration_seconds`, and `workqueue_work_duration_seconds`, so those three arrive as OTLP exponential histograms rather than fixed-bucket ones.

### Local Metrics with Prometheus (Kind Cluster)

For local development inside a `kind` cluster, Agent Substrate automatically provisions a Prometheus server in the `otel-system` namespace.
Expand Down
6 changes: 2 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ require (
github.com/containerd/ttrpc v1.2.8
github.com/envoyproxy/go-control-plane v0.14.0
github.com/envoyproxy/go-control-plane/envoy v1.37.0
github.com/go-logr/logr v1.4.3
github.com/google/go-cmp v0.7.0
github.com/google/go-containerregistry v0.21.7
github.com/google/nftables v0.3.0
Expand All @@ -34,6 +35,7 @@ require (
github.com/spiffe/go-spiffe/v2 v2.6.0
github.com/vishvananda/netlink v1.3.1
github.com/vishvananda/netns v0.0.5
go.opentelemetry.io/contrib/bridges/prometheus v0.68.0
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0
go.opentelemetry.io/otel v1.43.0
Expand Down Expand Up @@ -105,9 +107,7 @@ require (
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/fxamacker/cbor/v2 v2.9.0 // indirect
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-logr/zapr v1.3.0 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-openapi/jsonpointer v0.22.4 // indirect
github.com/go-openapi/jsonreference v0.21.4 // indirect
Expand Down Expand Up @@ -169,8 +169,6 @@ require (
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect
go.opentelemetry.io/proto/otlp v1.10.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.1 // indirect
go.yaml.in/yaml/v2 v2.4.4 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.52.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,8 @@ github.com/zeromq/goczmq v4.1.0+incompatible h1:cGVQaU6kIwwrGso0Pgbl84tzAz/h7FJ3
github.com/zeromq/goczmq v4.1.0+incompatible/go.mod h1:1uZybAJoSRCvZMH2rZxEwWBSmC4T7CB/xQOfChwPEzg=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/bridges/prometheus v0.68.0 h1:w3zlHYETbDwXyWHZlyyR58ZC39XGi8rAhkBgUgJ9d5w=
go.opentelemetry.io/contrib/bridges/prometheus v0.68.0/go.mod h1:GR/mClR2nn7vE8RLwxKjoBNg+QtgdDhRzxVa93koy5o=
go.opentelemetry.io/contrib/detectors/gcp v1.43.0 h1:62yY3dT7/ShwOxzA0RsKRgshBmfElKI4d/Myu2OxDFU=
go.opentelemetry.io/contrib/detectors/gcp v1.43.0/go.mod h1:RyaZMFY7yi1kAs45S6mbFGz8O8rqB0dTY14uzvG4LCs=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 h1:0Qx7VGBacMm9ZENQ7TnNObTYI4ShC+lHI16seduaxZo=
Expand Down
17 changes: 12 additions & 5 deletions internal/e2e/suites/metrics/metrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
"context"
"fmt"
"os"

"strings"
"testing"
"time"
Expand Down Expand Up @@ -95,7 +94,7 @@ func TestPlatformMetricsEmitted(t *testing.T) {

deadline := time.Now().Add(2 * time.Minute)
var missing []string
var ateomSeen bool
var ateomSeen, controllerSeen bool
var lastLabelErr error
for time.Now().Before(deadline) {
scrape, err := e2e.ScrapeCollectorMetrics(ctx)
Expand All @@ -104,7 +103,13 @@ func TestPlatformMetricsEmitted(t *testing.T) {
}
missing = e2e.MissingPlatformMetrics(scrape, e2e.PlatformMetricPrefixes)
ateomSeen = e2e.CollectorHasService(scrape, "ateom-gvisor", "ateom-microvm")
if len(missing) == 0 && ateomSeen {
// atecontroller bridges controller-runtime's Prometheus registry onto its OTLP
// reader, so the reconcile families are what prove the bridge, not just that
// some series arrived. Substring, not prefix: the collector's Prometheus
// exporter may re-suffix a name that already ends in _total.
controllerSeen = e2e.CollectorHasService(scrape, "atecontroller") &&
strings.Contains(scrape, "controller_runtime_")
if len(missing) == 0 && ateomSeen && controllerSeen {
// Verify ate_actor_crashes metric carries valid, non-empty low-cardinality labels for all attributes.
foundCrashLine := false
for _, line := range strings.Split(scrape, "\n") {
Expand Down Expand Up @@ -159,9 +164,11 @@ func TestPlatformMetricsEmitted(t *testing.T) {
}

if lastLabelErr != nil {
t.Fatalf("platform telemetry validation failed: missing metrics %v, ateom pushed=%v, error detail: %v", missing, ateomSeen, lastLabelErr)
t.Fatalf("platform telemetry validation failed: missing metrics %v, ateom pushed=%v, atecontroller pushed=%v, error detail: %v",
missing, ateomSeen, controllerSeen, lastLabelErr)
}
t.Fatalf("platform telemetry never reached the collector: missing metrics %v, ateom pushed=%v", missing, ateomSeen)
t.Fatalf("platform telemetry never reached the collector: missing metrics %v, ateom pushed=%v, atecontroller pushed=%v",
missing, ateomSeen, controllerSeen)
}

func triggerActorCrash(t *testing.T, ctx context.Context, clients *e2e.Clients, actorID string) {
Expand Down
Loading
Loading