diff --git a/pkg/bootstrap/client_v2_test.go b/pkg/bootstrap/client_v2_test.go index dca0045636..fd5beba6f5 100644 --- a/pkg/bootstrap/client_v2_test.go +++ b/pkg/bootstrap/client_v2_test.go @@ -7,6 +7,7 @@ import ( "crypto/x509" "net" "runtime" + "sync/atomic" "time" . "github.com/onsi/ginkgo/v2" @@ -65,7 +66,10 @@ var _ = Describe("Client V2", Ordered, Label("unit"), func() { Certificates: []tls.Certificate{*cert}, }))) - server := bootstrap.NewServerV2(store, cert.PrivateKey.(crypto.Signer)) + signer := cert.PrivateKey.(crypto.Signer) + ptr := &atomic.Pointer[crypto.Signer]{} + ptr.Store(&signer) + server := bootstrap.NewServerV2(store, ptr) bootstrapv2.RegisterBootstrapServer(srv, server) listener, err := net.Listen("tcp4", "127.0.0.1:0") diff --git a/pkg/bootstrap/server_v2.go b/pkg/bootstrap/server_v2.go index d566340635..9110812ea8 100644 --- a/pkg/bootstrap/server_v2.go +++ b/pkg/bootstrap/server_v2.go @@ -5,6 +5,7 @@ import ( "crypto" "fmt" "strings" + "sync/atomic" "maps" @@ -27,12 +28,12 @@ import ( type ServerV2 struct { bootstrapv2.UnsafeBootstrapServer - privateKey crypto.Signer + privateKey *atomic.Pointer[crypto.Signer] storage Storage clusterIdLocks storage.LockManager } -func NewServerV2(storage Storage, privateKey crypto.Signer) *ServerV2 { +func NewServerV2(storage Storage, privateKey *atomic.Pointer[crypto.Signer]) *ServerV2 { return &ServerV2{ privateKey: privateKey, storage: storage, @@ -84,7 +85,7 @@ func (h *ServerV2) Auth(ctx context.Context, authReq *bootstrapv2.BootstrapAuthR // Remove "Bearer " from the header bearerToken := strings.TrimSpace(strings.TrimPrefix(authHeader, "Bearer")) // Verify the token - payload, err := jws.Verify([]byte(bearerToken), jwa.EdDSA, h.privateKey.Public()) + payload, err := jws.Verify([]byte(bearerToken), jwa.EdDSA, (*h.privateKey.Load()).Public()) if err != nil { return nil, util.StatusError(codes.PermissionDenied) } diff --git a/pkg/bootstrap/server_v2_test.go b/pkg/bootstrap/server_v2_test.go index 1ba7b67660..970d4f174b 100644 --- a/pkg/bootstrap/server_v2_test.go +++ b/pkg/bootstrap/server_v2_test.go @@ -9,6 +9,7 @@ import ( "errors" "net" "strings" + "sync/atomic" "time" "github.com/lestrrat-go/jwx/jwa" @@ -65,13 +66,16 @@ var _ = Describe("Server V2", Ordered, Label("unit"), func() { Expect(err).NotTo(HaveOccurred()) cert = &crt + ptr := &atomic.Pointer[crypto.Signer]{} + signer := cert.PrivateKey.(crypto.Signer) + ptr.Store(&signer) srv := grpc.NewServer(grpc.Creds(insecure.NewCredentials())) server := bootstrap.NewServerV2(bootstrap.StorageConfig{ TokenStore: mockTokenStore, ClusterStore: mockClusterStore, KeyringStoreBroker: mockKeyringStoreBroker, LockManagerBroker: inmemory.NewLockManagerBroker(), - }, cert.PrivateKey.(crypto.Signer)) + }, ptr) bootstrapv2.RegisterBootstrapServer(srv, server) listener := bufconn.Listen(1024 * 1024) diff --git a/pkg/gateway/connections.go b/pkg/gateway/connections.go index 98176991f3..d6f6c08975 100644 --- a/pkg/gateway/connections.go +++ b/pkg/gateway/connections.go @@ -3,28 +3,37 @@ package gateway import ( "context" "encoding/base64" + "fmt" "io" "log/slog" + "os" "strings" sync "sync" "time" corev1 "github.com/rancher/opni/pkg/apis/core/v1" "github.com/rancher/opni/pkg/auth/cluster" + "github.com/rancher/opni/pkg/config/reactive" + configv1 "github.com/rancher/opni/pkg/config/v1" "github.com/rancher/opni/pkg/logger" "github.com/rancher/opni/pkg/storage" "github.com/rancher/opni/pkg/storage/lock" "github.com/rancher/opni/pkg/util" "github.com/rancher/opni/pkg/util/streams" + "github.com/rancher/opni/pkg/versions" + "github.com/samber/lo" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protopath" ) type TrackedConnectionListener interface { // Called when a new agent connection to any gateway instance is tracked. // The provided context will be canceled when the tracked connection is deleted. + // If a tracked connection is updated, this method will be called again with + // the same context, agentId, and leaseId, but updated instanceInfo. // Implementations of this method MUST NOT block. HandleTrackedConnection(ctx context.Context, agentId string, leaseId string, instanceInfo *corev1.InstanceInfo) } @@ -62,11 +71,11 @@ type activeTrackedInstance struct { // All gateway instances track the same state and update in real time at the // speed of the underlying storage backend. type ConnectionTracker struct { - localInstanceInfo *corev1.InstanceInfo - rootContext context.Context - kv storage.KeyValueStore - lm storage.LockManager - logger *slog.Logger + rootContext context.Context + kv storage.KeyValueStore + lm storage.LockManager + mgr *configv1.GatewayConfigManager + logger *slog.Logger listenersMu sync.Mutex connListeners []TrackedConnectionListener @@ -75,24 +84,39 @@ type ConnectionTracker struct { mu sync.RWMutex activeConnections map[string]*activeTrackedConnection activeInstances map[string]*activeTrackedInstance + + localInstanceInfoMu sync.Mutex + localInstanceInfo *corev1.InstanceInfo + // relayConf reactive.Reactive[*configv1.RelayServerSpec] + // managementConf reactive.Reactive[*configv1.ManagementServerSpec] + // dashboardConf reactive.Reactive[*configv1.DashboardServerSpec] } func NewConnectionTracker( rootContext context.Context, - localInstanceInfo *corev1.InstanceInfo, + mgr *configv1.GatewayConfigManager, kv storage.KeyValueStore, lm storage.LockManager, lg *slog.Logger, ) *ConnectionTracker { - return &ConnectionTracker{ - localInstanceInfo: localInstanceInfo, + hostname, _ := os.Hostname() + ct := &ConnectionTracker{ + localInstanceInfo: &corev1.InstanceInfo{ + Annotations: map[string]string{ + "hostname": hostname, + "pid": fmt.Sprint(os.Getpid()), + "version": versions.Version, + }, + }, rootContext: rootContext, + mgr: mgr, kv: kv, lm: lm, logger: lg, activeConnections: make(map[string]*activeTrackedConnection), activeInstances: make(map[string]*activeTrackedInstance), } + return ct } func NewReadOnlyConnectionTracker( @@ -153,6 +177,8 @@ func (ct *ConnectionTracker) AddTrackedInstanceListener(listener TrackedInstance } func (ct *ConnectionTracker) LocalInstanceInfo() *corev1.InstanceInfo { + ct.localInstanceInfoMu.Lock() + defer ct.localInstanceInfoMu.Unlock() return util.ProtoClone(ct.localInstanceInfo) } @@ -160,7 +186,7 @@ func (ct *ConnectionTracker) LocalInstanceInfo() *corev1.InstanceInfo { // and the underlying kv store watcher is closed. func (ct *ConnectionTracker) Run(ctx context.Context) error { var wg sync.WaitGroup - if ct.localInstanceInfo != nil { + if ct.LocalInstanceInfo() != nil { wg.Add(1) go func() { defer wg.Done() @@ -199,16 +225,76 @@ func (ct *ConnectionTracker) Run(ctx context.Context) error { // "$instances". This key can be used to track the set of all instances // that are currently running. func (ct *ConnectionTracker) lockInstance(ctx context.Context) { - instanceInfo := &corev1.InstanceInfo{ - RelayAddress: ct.localInstanceInfo.GetRelayAddress(), - ManagementAddress: ct.localInstanceInfo.GetManagementAddress(), - GatewayAddress: ct.localInstanceInfo.GetGatewayAddress(), - WebAddress: ct.localInstanceInfo.GetWebAddress(), - } + ctx, ca := context.WithCancel(ctx) + defer ca() + + locker := ct.lm.Locker(instancesKey, lock.WithAcquireContext(ctx), + lock.WithInitialValue(base64.StdEncoding.EncodeToString(util.Must(proto.Marshal(ct.LocalInstanceInfo()))))) + + updateInstanceInfo, cancel := lo.NewDebounce(100*time.Millisecond, func() { + ct.kv.Put(ctx, locker.Key(), util.Must(proto.Marshal(ct.LocalInstanceInfo()))) + }) + defer cancel() + + listenOnce := sync.OnceFunc(func() { + relay := reactive.Message[*configv1.RelayServerSpec](ct.mgr.Reactive(protopath.Path(configv1.ProtoPath().Relay()))) + mgmt := reactive.Message[*configv1.ManagementServerSpec](ct.mgr.Reactive(protopath.Path(configv1.ProtoPath().Management()))) + server := reactive.Message[*configv1.ServerSpec](ct.mgr.Reactive(protopath.Path(configv1.ProtoPath().Server()))) + dashboard := reactive.Message[*configv1.DashboardServerSpec](ct.mgr.Reactive(protopath.Path(configv1.ProtoPath().Dashboard()))) + + relay.WatchFunc(ctx, func(msg *configv1.RelayServerSpec) { + ct.localInstanceInfoMu.Lock() + defer ct.localInstanceInfoMu.Unlock() + if addr := msg.GetAdvertiseAddress(); addr != "" { + ct.localInstanceInfo.RelayAddress = addr + } else { + ct.logger.Warn("relay advertise address not set; will advertise the listen address") + ct.localInstanceInfo.RelayAddress = msg.GetGrpcListenAddress() + } + updateInstanceInfo() + }) + + mgmt.WatchFunc(ctx, func(msg *configv1.ManagementServerSpec) { + ct.localInstanceInfoMu.Lock() + defer ct.localInstanceInfoMu.Unlock() + if addr := msg.GetAdvertiseAddress(); addr != "" { + ct.localInstanceInfo.ManagementAddress = addr + } else { + ct.logger.Warn("management advertise address not set; will advertise the listen address") + ct.localInstanceInfo.ManagementAddress = msg.GetGrpcListenAddress() + } + updateInstanceInfo() + }) + + server.WatchFunc(ctx, func(msg *configv1.ServerSpec) { + ct.localInstanceInfoMu.Lock() + defer ct.localInstanceInfoMu.Unlock() + if addr := msg.GetAdvertiseAddress(); addr != "" { + ct.localInstanceInfo.GatewayAddress = addr + } else { + ct.logger.Warn("gateway advertise address not set; will advertise the listen address") + ct.localInstanceInfo.GatewayAddress = msg.GetGrpcListenAddress() + } + updateInstanceInfo() + }) + + dashboard.WatchFunc(ctx, func(msg *configv1.DashboardServerSpec) { + ct.localInstanceInfoMu.Lock() + defer ct.localInstanceInfoMu.Unlock() + if addr := msg.GetAdvertiseAddress(); addr != "" { + ct.localInstanceInfo.WebAddress = addr + } else { + ct.logger.Warn("web advertise address not set; will advertise the listen address") + ct.localInstanceInfo.WebAddress = msg.GetHttpListenAddress() + } + updateInstanceInfo() + }) + }) for ctx.Err() == nil { locker := ct.lm.Locker(instancesKey, lock.WithAcquireContext(ctx), - lock.WithInitialValue(base64.StdEncoding.EncodeToString(util.Must(proto.Marshal(instanceInfo))))) + lock.WithInitialValue(base64.StdEncoding.EncodeToString(util.Must(proto.Marshal(ct.LocalInstanceInfo()))))) locker.Lock() + listenOnce() } } @@ -219,12 +305,7 @@ var instanceInfoKey = instanceInfoKeyType{} func (ct *ConnectionTracker) StreamServerInterceptor() grpc.StreamServerInterceptor { return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { agentId := cluster.StreamAuthorizedID(ss.Context()) - instanceInfo := &corev1.InstanceInfo{ - RelayAddress: ct.localInstanceInfo.GetRelayAddress(), - ManagementAddress: ct.localInstanceInfo.GetManagementAddress(), - GatewayAddress: ct.localInstanceInfo.GetGatewayAddress(), - WebAddress: ct.localInstanceInfo.GetWebAddress(), - } + instanceInfo := ct.LocalInstanceInfo() locker := ct.lm.Locker(agentId, lock.WithAcquireContext(ss.Context()), lock.WithInitialValue(base64.StdEncoding.EncodeToString(util.Must(proto.Marshal(instanceInfo)))), @@ -295,54 +376,62 @@ func (ct *ConnectionTracker) handleConnectionEventLocked(event storage.WatchEven "agentId", agentId, "key", event.Current.Key(), ) + info, err := instanceInfo() + if err != nil { + lg.With(logger.Err(err)).Error("failed to unmarshal instance info") + return + } + isUpdate := false if conn, ok := ct.activeConnections[agentId]; ok { - if conn.leaseId == leaseId { - // key was updated, not a new connection - lg.Debug("tracked connection updated") - return - } - info, err := instanceInfo() - if err != nil { - lg.With(logger.Err(err)).Error("failed to unmarshal instance info") - return - } if !info.GetAcquired() { // a different instance is only attempting to acquire the lock, // ignore the event ct.logger.With("agent", agentId, "instance", info.GetRelayAddress()).Debug("observed lock attempt from another instance") return } - // a different instance has acquired the lock, invalidate - // the current tracked connection - ct.logger.With("agentId", agentId).Debug("tracked connection invalidated") - conn.cancelTrackingContext() - delete(ct.activeConnections, agentId) - } - info, err := instanceInfo() - if err != nil { - lg.With(logger.Err(err)).Error("failed to unmarshal instance info") - return + if conn.leaseId == leaseId { + // key was updated, not a new connection + isUpdate = true + } else { + // a different instance has acquired the lock, invalidate + // the current tracked connection + ct.logger.With("agentId", agentId).Debug("tracked connection invalidated") + conn.cancelTrackingContext() + delete(ct.activeConnections, agentId) + } } if !info.GetAcquired() { return // ignore unacquired connections } if ct.IsLocalInstance(info) { ct.logger.With("agentId", agentId).Debug("tracking new connection (local)") + } else if isUpdate { + ct.logger.With("agentId", agentId).Debug("tracked connection updated") } else { ct.logger.With("agentId", agentId).Debug("tracking new connection") } - ctx, cancel := context.WithCancel(ct.rootContext) - conn := &activeTrackedConnection{ - agentId: agentId, - leaseId: leaseId, - revision: event.Current.Revision(), - instanceInfo: info, - trackingContext: ctx, - cancelTrackingContext: cancel, + + var trackingContext context.Context + if !isUpdate { + ctx, cancel := context.WithCancel(ct.rootContext) + trackingContext = ctx + conn := &activeTrackedConnection{ + agentId: agentId, + leaseId: leaseId, + revision: event.Current.Revision(), + instanceInfo: info, + trackingContext: ctx, + cancelTrackingContext: cancel, + } + ct.activeConnections[agentId] = conn + } else { + conn := ct.activeConnections[agentId] + conn.revision = event.Current.Revision() + conn.instanceInfo = info + trackingContext = conn.trackingContext } - ct.activeConnections[agentId] = conn for _, listener := range ct.connListeners { - listener.HandleTrackedConnection(ctx, agentId, leaseId, info) + listener.HandleTrackedConnection(trackingContext, agentId, leaseId, info) } case storage.WatchEventDelete: agentId, leaseId, ok := decodeKey(event.Previous.Key()) @@ -389,7 +478,9 @@ func (ct *ConnectionTracker) handleInstanceEventLocked(event storage.WatchEvent[ ct.logger.With(logger.Err(err)).Error("failed to unmarshal instance info") return } - if _, ok := ct.activeInstances[leaseId]; ok { + if existing, ok := ct.activeInstances[leaseId]; ok { + ct.logger.With("leaseId", leaseId).Debug("tracked instance updated") + existing.instanceInfo = instanceInfo return } ct.logger.With("leaseId", leaseId).Debug("tracking new instance") diff --git a/pkg/gateway/delegate.go b/pkg/gateway/delegate.go index 277e7ca230..8c5aa92aff 100644 --- a/pkg/gateway/delegate.go +++ b/pkg/gateway/delegate.go @@ -3,6 +3,7 @@ package gateway import ( "context" "math" + "net" "sync" "time" @@ -15,7 +16,8 @@ import ( managementv1 "github.com/rancher/opni/pkg/apis/management/v1" streamv1 "github.com/rancher/opni/pkg/apis/stream/v1" "github.com/rancher/opni/pkg/auth/cluster" - "github.com/rancher/opni/pkg/config/v1beta1" + "github.com/rancher/opni/pkg/config/reactive" + configv1 "github.com/rancher/opni/pkg/config/v1" "github.com/rancher/opni/pkg/logger" "github.com/rancher/opni/pkg/storage" "github.com/rancher/opni/pkg/util" @@ -31,7 +33,7 @@ import ( type DelegateServer struct { DelegateServerOptions uuid string - config v1beta1.GatewayConfigSpec + mgr *configv1.GatewayConfigManager mu *sync.RWMutex rcond *sync.Cond localAgents map[string]grpc.ClientConnInterface @@ -59,14 +61,19 @@ func WithConnectionTracker(connectionTracker *ConnectionTracker) DelegateServerO } } -func NewDelegateServer(config v1beta1.GatewayConfigSpec, clusterStore storage.ClusterStore, lg *slog.Logger, opts ...DelegateServerOption) *DelegateServer { +func NewDelegateServer( + mgr *configv1.GatewayConfigManager, + clusterStore storage.ClusterStore, + lg *slog.Logger, + opts ...DelegateServerOption, +) *DelegateServer { options := DelegateServerOptions{} options.apply(opts...) mu := &sync.RWMutex{} return &DelegateServer{ DelegateServerOptions: options, uuid: uuid.NewString(), - config: config, + mgr: mgr, mu: mu, rcond: sync.NewCond(mu.RLocker()), localAgents: make(map[string]grpc.ClientConnInterface), @@ -238,7 +245,8 @@ func (d *DelegateServer) NewRelayServer() *RelayServer { } return &RelayServer{ delegateSrv: d, - listenAddress: d.config.Management.RelayListenAddress, + mgr: d.mgr, + listenAddress: reactive.Scalar[string](d.mgr.Reactive(configv1.ProtoPath().Relay().GrpcListenAddress())), connectionTracker: d.ct, logger: d.logger.WithGroup("relay"), } @@ -246,17 +254,43 @@ func (d *DelegateServer) NewRelayServer() *RelayServer { type RelayServer struct { connectionTracker *ConnectionTracker - listenAddress string + mgr *configv1.GatewayConfigManager + listenAddress reactive.Reactive[string] delegateSrv *DelegateServer logger *slog.Logger } -func (rs *RelayServer) ListenAndServe(ctx context.Context) error { - listener, err := util.NewProtocolListener(rs.listenAddress) - if err != nil { - return err - } +func (rs *RelayServer) ListenAndServe(ctx context.Context) (err error) { + var cancel context.CancelFunc + var done chan struct{} + rs.listenAddress.WatchFunc(ctx, func(listenAddress string) { + if cancel != nil { + cancel() + <-done + } + ctx, cancel = context.WithCancel(ctx) + done = make(chan struct{}) + + var listener net.Listener + listener, err = util.NewProtocolListener(listenAddress) + if err != nil { + return + } + go func() { + defer close(done) + err := rs.serve(ctx, listener) + if err != nil { + rs.logger.With( + logger.Err(err), + ).Error("relay server exited with error") + } + }() + }) + <-ctx.Done() + return // last error +} +func (rs *RelayServer) serve(ctx context.Context, listener net.Listener) error { creds := insecure.NewCredentials() // todo? server := grpc.NewServer( diff --git a/pkg/gateway/gateway.go b/pkg/gateway/gateway.go index 66c9616238..2577c8d340 100644 --- a/pkg/gateway/gateway.go +++ b/pkg/gateway/gateway.go @@ -6,8 +6,8 @@ import ( "crypto/tls" "errors" "fmt" - "os" "slices" + "sync/atomic" "log/slog" @@ -23,9 +23,9 @@ import ( "github.com/rancher/opni/pkg/auth/session" "github.com/rancher/opni/pkg/bootstrap" "github.com/rancher/opni/pkg/capabilities" - "github.com/rancher/opni/pkg/config" - cfgmeta "github.com/rancher/opni/pkg/config/meta" - "github.com/rancher/opni/pkg/config/v1beta1" + "github.com/rancher/opni/pkg/config/reactive" + "github.com/rancher/opni/pkg/config/reactive/subtle" + configv1 "github.com/rancher/opni/pkg/config/v1" "github.com/rancher/opni/pkg/health" "github.com/rancher/opni/pkg/keyring" "github.com/rancher/opni/pkg/logger" @@ -38,23 +38,23 @@ import ( "github.com/rancher/opni/pkg/storage" "github.com/rancher/opni/pkg/update" k8sserver "github.com/rancher/opni/pkg/update/kubernetes/server" + "github.com/rancher/opni/pkg/update/patch" patchserver "github.com/rancher/opni/pkg/update/patch/server" "github.com/rancher/opni/pkg/util" - "github.com/rancher/opni/pkg/versions" "github.com/samber/lo" "github.com/spf13/afero" "golang.org/x/mod/module" "google.golang.org/grpc" "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials" "google.golang.org/grpc/status" + "google.golang.org/protobuf/reflect/protopath" "google.golang.org/protobuf/types/known/emptypb" ) type Gateway struct { GatewayOptions - config *config.GatewayConfig - tlsConfig *tls.Config + mgr *configv1.GatewayConfigManager + certs reactive.Reactive[*configv1.CertsSpec] httpServer *GatewayHTTPServer grpcServer *GatewayGRPCServer connectionTracker *ConnectionTracker @@ -67,7 +67,6 @@ type Gateway struct { type GatewayOptions struct { logger *slog.Logger - lifecycler config.Lifecycler extraUpdateHandlers []update.UpdateTypeHandler } @@ -79,12 +78,6 @@ func (o *GatewayOptions) apply(opts ...GatewayOption) { } } -func WithLifecycler(lc config.Lifecycler) GatewayOption { - return func(o *GatewayOptions) { - o.lifecycler = lc - } -} - func WithExtraUpdateHandlers(handlers ...update.UpdateTypeHandler) GatewayOption { return func(o *GatewayOptions) { o.extraUpdateHandlers = append(o.extraUpdateHandlers, handlers...) @@ -97,23 +90,19 @@ func WithLogger(logger *slog.Logger) GatewayOption { } } -func NewGateway(ctx context.Context, conf *config.GatewayConfig, pl plugins.LoaderInterface, opts ...GatewayOption) *Gateway { +func NewGateway( + ctx context.Context, + mgr *configv1.GatewayConfigManager, + storageBackend storage.Backend, + pl plugins.LoaderInterface, + opts ...GatewayOption, +) *Gateway { options := GatewayOptions{ - lifecycler: config.NewUnavailableLifecycler(cfgmeta.ObjectList{conf}), - logger: logger.New().WithGroup("gateway"), + logger: logger.New().WithGroup("gateway"), } options.apply(opts...) lg := options.logger - conf.Spec.SetDefaults() - - storageBackend, err := machinery.ConfigureStorageBackend(ctx, &conf.Spec.Storage) - if err != nil { - lg.With( - logger.Err(err), - ).Error("failed to configure storage backend") - panic("failed to configure storage backend") - } capBackendStore := capabilities.NewBackendStore(lg) @@ -175,12 +164,21 @@ func NewGateway(ctx context.Context, conf *config.GatewayConfig, pl plugins.Load })) // set up http server - httpServer := NewHTTPServer(ctx, &conf.Spec, lg, pl) + httpServer, err := NewHTTPServer(ctx, mgr, lg, pl) + if err != nil { + panic(fmt.Errorf("failed to create http server: %w", err)) + } + + runtimeKeyDirList := subtle.WaitOne(ctx, mgr.Reactive(configv1.ProtoPath().Keyring().RuntimeKeyDirs())).List() + runtimeKeyDirs := make([]string, 0, runtimeKeyDirList.Len()) + for i, l := 0, runtimeKeyDirList.Len(); i < l; i++ { + runtimeKeyDirs = append(runtimeKeyDirs, runtimeKeyDirList.Get(i).String()) + } // Set up cluster auth ephemeralKeys, err := machinery.LoadEphemeralKeys(afero.Afero{ Fs: afero.NewOsFs(), - }, conf.Spec.Keyring.EphemeralKeyDirs...) + }, runtimeKeyDirs...) if err != nil { lg.With( logger.Err(err), @@ -212,7 +210,10 @@ func NewGateway(ctx context.Context, conf *config.GatewayConfig, pl plugins.Load updateServer := update.NewUpdateServer(lg) // set up plugin sync server - binarySyncServer, err := patchserver.NewFilesystemPluginSyncServer(conf.Spec.Plugins, lg, + pluginConfig := subtle.WaitOneMessage[*configv1.PluginsSpec](ctx, mgr.Reactive(protopath.Path(configv1.ProtoPath().Plugins()))) + patchEngine := patch.NewReactivePatchEngine(ctx, lg, mgr.Reactive(configv1.ProtoPath().Upgrades().Plugins().Binary().PatchEngine())) + + binarySyncServer, err := patchserver.NewFilesystemPluginSyncServer(ctx, pluginConfig.Cache, patchEngine, lg, patchserver.WithPluginSyncFilters(func(pm meta.PluginMeta) bool { if pm.ExtendedMetadata != nil { // only sync plugins that have the agent mode set @@ -236,14 +237,17 @@ func NewGateway(ctx context.Context, conf *config.GatewayConfig, pl plugins.Load updateServer.RegisterUpdateHandler(binarySyncServer.Strategy(), binarySyncServer) - kubernetesSyncServer, err := k8sserver.NewKubernetesSyncServer(conf.Spec.AgentUpgrades.Kubernetes, lg) + k8sSyncServerConfig := reactive.Message[*configv1.KubernetesAgentUpgradeSpec]( + mgr.Reactive(protopath.Path(configv1.ProtoPath().Upgrades().Agents().Kubernetes()))) + kubernetesSyncServer, err := k8sserver.NewKubernetesSyncServer(ctx, k8sSyncServerConfig, lg) if err != nil { lg.With( logger.Err(err), ).Error("failed to create kubernetes agent sync server") - panic("failed to create kubernetes agent sync server") + // panic("failed to create kubernetes agent sync server") + } else { + updateServer.RegisterUpdateHandler(kubernetesSyncServer.Strategy(), kubernetesSyncServer) } - updateServer.RegisterUpdateHandler(kubernetesSyncServer.Strategy(), kubernetesSyncServer) for _, handler := range options.extraUpdateHandlers { updateServer.RegisterUpdateHandler(handler.Strategy(), handler) @@ -252,19 +256,19 @@ func NewGateway(ctx context.Context, conf *config.GatewayConfig, pl plugins.Load httpServer.metricsRegisterer.MustRegister(updateServer.Collectors()...) // set up grpc server - tlsConfig, pkey, err := grpcTLSConfig(&conf.Spec) - if err != nil { - lg.With( - logger.Err(err), - ).Error("failed to load TLS config") - panic("failed to load TLS config") - } - - rateLimitOpts := []RatelimiterOption{} - if conf.Spec.RateLimit != nil { - rateLimitOpts = append(rateLimitOpts, WithRate(conf.Spec.RateLimit.Rate)) - rateLimitOpts = append(rateLimitOpts, WithBurst(conf.Spec.RateLimit.Burst)) - } + // tlsConfig, pkey, err := grpcTLSConfig(&conf.Spec) + // if err != nil { + // lg.With( + // logger.Err(err), + // ).Error("failed to load TLS config") + // panic("failed to load TLS config") + // } + + // rateLimitOpts := []RatelimiterOption{} + // if conf.RateLimiting != nil { + // rateLimitOpts = append(rateLimitOpts, WithRate(conf.RateLimiting.GetRate())) + // rateLimitOpts = append(rateLimitOpts, WithBurst(int(conf.RateLimiting.GetBurst()))) + // } // set up stream server listener := health.NewListener() @@ -272,47 +276,13 @@ func NewGateway(ctx context.Context, conf *config.GatewayConfig, pl plugins.Load // set up connection tracker var connectionTracker *ConnectionTracker - if conf.Spec.Management.RelayListenAddress != "" { - // require a lock manager - storageBackendLmBroker, ok := storageBackend.(storage.LockManagerBroker) - if !ok || storageBackendLmBroker == nil { - panic("storage backend does not support distributed locking and cannot be used if the relay server is configured") - } + { + storageBackendLmBroker := storageBackend.(storage.LockManagerBroker) lg := lg.WithGroup("connections") connectionsKv := storageBackend.KeyValueStore("connections") connectionsLm := storageBackendLmBroker.LockManager("connections") - relayAdvertiseAddr := conf.Spec.Management.RelayAdvertiseAddress - mgmtAdvertiseAddr := conf.Spec.Management.GRPCAdvertiseAddress - gatewayAdvertiseAddr := conf.Spec.GRPCAdvertiseAddress - webAdvertiseAddr := conf.Spec.Management.WebAdvertiseAddress - if relayAdvertiseAddr == "" { - lg.Warn("relay advertise address not set; will advertise the listen address") - relayAdvertiseAddr = conf.Spec.Management.RelayListenAddress - } - if mgmtAdvertiseAddr == "" { - lg.Warn("management advertise address not set; will advertise the listen address") - mgmtAdvertiseAddr = conf.Spec.Management.GRPCListenAddress - } - if gatewayAdvertiseAddr == "" { - lg.Warn("gateway advertise address not set; will advertise the listen address") - gatewayAdvertiseAddr = conf.Spec.GRPCListenAddress - } - hostname, _ := os.Hostname() - if webAdvertiseAddr == "" { - lg.Warn("web advertise address not set; will advertise the listen address") - webAdvertiseAddr = conf.Spec.Management.WebListenAddress - } - connectionTracker = NewConnectionTracker(ctx, &corev1.InstanceInfo{ - RelayAddress: os.ExpandEnv(relayAdvertiseAddr), - ManagementAddress: os.ExpandEnv(mgmtAdvertiseAddr), - GatewayAddress: os.ExpandEnv(gatewayAdvertiseAddr), - WebAddress: os.ExpandEnv(webAdvertiseAddr), - Annotations: map[string]string{ - "hostname": hostname, - "pid": fmt.Sprint(os.Getpid()), - "version": versions.Version, - }, - }, connectionsKv, connectionsLm, lg) + + connectionTracker = NewConnectionTracker(ctx, mgr, connectionsKv, connectionsLm, lg) writerManager := NewHealthStatusWriterManager(ctx, connectionsKv, lg) go health.Copy(ctx, writerManager, listener) @@ -324,8 +294,6 @@ func NewGateway(ctx context.Context, conf *config.GatewayConfig, pl plugins.Load panic(fmt.Errorf("failed to create health status reader: %w", err)) } go health.Copy(ctx, buffer, kvReader) - } else { - go health.Copy(ctx, buffer, listener) } monitor := health.NewMonitor(health.WithLogger(lg.WithGroup("monitor"))) @@ -333,7 +301,7 @@ func NewGateway(ctx context.Context, conf *config.GatewayConfig, pl plugins.Load if connectionTracker != nil { delegateServerOpts = append(delegateServerOpts, WithConnectionTracker(connectionTracker)) } - delegate := NewDelegateServer(conf.Spec, storageBackend, lg, delegateServerOpts...) + delegate := NewDelegateServer(mgr, storageBackend, lg, delegateServerOpts...) agentHandler := MultiConnectionHandler(listener, delegate) capDataSource := &capabilitiesDataSource{ @@ -347,7 +315,8 @@ func NewGateway(ctx context.Context, conf *config.GatewayConfig, pl plugins.Load // initialize grpc server var streamInterceptors []grpc.StreamServerInterceptor - streamInterceptors = append(streamInterceptors, NewRateLimiterInterceptor(lg, rateLimitOpts...).StreamServerInterceptor()) + rateLimitConfig := reactive.Message[*configv1.RateLimitingSpec](mgr.Reactive(protopath.Path(configv1.ProtoPath().RateLimiting()))) + streamInterceptors = append(streamInterceptors, NewRateLimiterInterceptor(ctx, lg, rateLimitConfig).StreamServerInterceptor()) streamInterceptors = append(streamInterceptors, clusterAuth) if connectionTracker != nil { streamInterceptors = append(streamInterceptors, connectionTracker.StreamServerInterceptor()) @@ -355,8 +324,7 @@ func NewGateway(ctx context.Context, conf *config.GatewayConfig, pl plugins.Load streamInterceptors = append(streamInterceptors, updateServer.StreamServerInterceptor()) streamInterceptors = append(streamInterceptors, NewLastKnownDetailsApplier(storageBackend)) - grpcServer := NewGRPCServer(&conf.Spec, lg, - grpc.Creds(credentials.NewTLS(tlsConfig)), + grpcServer := NewGRPCServer(mgr, lg, grpc.ChainStreamInterceptor(streamInterceptors...), ) @@ -369,14 +337,21 @@ func NewGateway(ctx context.Context, conf *config.GatewayConfig, pl plugins.Load pl.Hook(hooks.OnLoadMC(streamSvc.OnPluginLoad)) + certs := reactive.Message[*configv1.CertsSpec](mgr.Reactive(protopath.Path(configv1.ProtoPath().Certs()))) // set up bootstrap server + pkey := &atomic.Pointer[crypto.Signer]{} + certs.WatchFunc(ctx, func(value *configv1.CertsSpec) { + tlsConfig, _ := value.AsTlsConfig(tls.NoClientCert) + signer := tlsConfig.Certificates[0].PrivateKey.(crypto.Signer) + pkey.Store(&signer) + }) bootstrapServerV2 := bootstrap.NewServerV2(bootstrap.NewStorage(storageBackend), pkey) bootstrapv2.RegisterBootstrapServer(grpcServer, bootstrapServerV2) g := &Gateway{ + certs: certs, + mgr: mgr, GatewayOptions: options, - tlsConfig: tlsConfig, - config: conf, storageBackend: storageBackend, capDataSource: capDataSource, httpServer: httpServer, @@ -396,7 +371,6 @@ type keyValueStoreServer interface { func (g *Gateway) ListenAndServe(ctx context.Context) error { lg := g.logger ctx, ca := context.WithCancelCause(ctx) - channels := []<-chan error{ // start http server lo.Async(func() error { @@ -450,7 +424,8 @@ func (g *Gateway) ListenAndServe(ctx context.Context) error { })) } - return util.WaitAll(ctx, ca, channels...) + util.WaitAll(ctx, ca, channels...) + return context.Cause(ctx) } // Implements management.CoreDataSource @@ -460,7 +435,11 @@ func (g *Gateway) StorageBackend() storage.Backend { // Implements management.CoreDataSource func (g *Gateway) TLSConfig() *tls.Config { - return g.httpServer.tlsConfig + tc, err := g.certs.Value().AsTlsConfig(tls.NoClientCert) + if err != nil { + return nil + } + return tc } // Implements management.CapabilitiesDataSource @@ -485,30 +464,3 @@ func (g *Gateway) WatchClusterHealthStatus(ctx context.Context) <-chan *corev1.C func (g *Gateway) MustRegisterCollector(collector prometheus.Collector) { g.httpServer.metricsRegisterer.MustRegister(collector) } - -func grpcTLSConfig(cfg *v1beta1.GatewayConfigSpec) (*tls.Config, crypto.Signer, error) { - servingCertBundle, caPool, err := util.LoadServingCertBundle(cfg.Certs) - if err != nil { - return nil, nil, err - } - return &tls.Config{ - MinVersion: tls.VersionTLS13, - RootCAs: caPool, - Certificates: []tls.Certificate{*servingCertBundle}, - ClientAuth: tls.NoClientCert, - }, servingCertBundle.PrivateKey.(crypto.Signer), nil -} - -func httpTLSConfig(cfg *v1beta1.GatewayConfigSpec) (*tls.Config, crypto.Signer, error) { - servingCertBundle, caPool, err := util.LoadServingCertBundle(cfg.Certs) - if err != nil { - return nil, nil, err - } - return &tls.Config{ - MinVersion: tls.VersionTLS12, - RootCAs: caPool, - ClientCAs: caPool, - Certificates: []tls.Certificate{*servingCertBundle}, - ClientAuth: tls.RequireAndVerifyClientCert, - }, servingCertBundle.PrivateKey.(crypto.Signer), nil -} diff --git a/pkg/gateway/grpc.go b/pkg/gateway/grpc.go index 61b64d1a62..be786032e9 100644 --- a/pkg/gateway/grpc.go +++ b/pkg/gateway/grpc.go @@ -2,6 +2,7 @@ package gateway import ( "context" + "crypto/tls" "net" "runtime" "sync" @@ -12,13 +13,17 @@ import ( "github.com/samber/lo" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/health" healthv1 "google.golang.org/grpc/health/grpc_health_v1" "google.golang.org/grpc/keepalive" + "google.golang.org/protobuf/reflect/protopath" + "google.golang.org/protobuf/reflect/protoreflect" "github.com/rancher/opni/pkg/agent" streamv1 "github.com/rancher/opni/pkg/apis/stream/v1" - "github.com/rancher/opni/pkg/config/v1beta1" + "github.com/rancher/opni/pkg/config/reactive" + configv1 "github.com/rancher/opni/pkg/config/v1" "github.com/rancher/opni/pkg/util" ) @@ -50,7 +55,7 @@ func (f ConnectionHandlerFunc) HandleAgentConnection(ctx context.Context, client type GatewayGRPCServer struct { streamv1.UnsafeStreamServer - conf *v1beta1.GatewayConfigSpec + mgr *configv1.GatewayConfigManager logger *slog.Logger serverOpts []grpc.ServerOption @@ -59,23 +64,62 @@ type GatewayGRPCServer struct { } func NewGRPCServer( - cfg *v1beta1.GatewayConfigSpec, + mgr *configv1.GatewayConfigManager, lg *slog.Logger, opts ...grpc.ServerOption, ) *GatewayGRPCServer { return &GatewayGRPCServer{ - conf: cfg, + mgr: mgr, logger: lg.WithGroup("grpc"), serverOpts: opts, } } -func (s *GatewayGRPCServer) ListenAndServe(ctx context.Context) error { - listener, err := net.Listen("tcp4", s.conf.GRPCListenAddress) - if err != nil { - return err +func (s *GatewayGRPCServer) ListenAndServe(ctx context.Context) (serveError error) { + var cancel context.CancelFunc + var done chan struct{} + doServe := func(addr string, certs *configv1.CertsSpec) error { + if cancel != nil { + cancel() + select { + case <-ctx.Done(): + return nil + case <-done: + // server stopped, continue + } + } + done = make(chan struct{}) + var serveCtx context.Context + serveCtx, cancel = context.WithCancel(ctx) + listener, err := net.Listen("tcp4", addr) + if err != nil { + return err + } + tlsConfig, err := certs.AsTlsConfig(tls.NoClientCert) + if err != nil { + return err + } + go func() { + s.serve(serveCtx, listener, tlsConfig) + close(done) + }() + return nil } + + reactive.Bind(ctx, + func(v []protoreflect.Value) { + serveError = doServe(v[0].String(), v[1].Message().Interface().(*configv1.CertsSpec)) + }, + s.mgr.Reactive(configv1.ProtoPath().Server().GrpcListenAddress()), + s.mgr.Reactive(protopath.Path(configv1.ProtoPath().Certs())), + ) + <-ctx.Done() + return +} + +func (s *GatewayGRPCServer) serve(ctx context.Context, listener net.Listener, tlsConfig *tls.Config) error { server := grpc.NewServer(append(s.serverOpts, + grpc.Creds(credentials.NewTLS(tlsConfig)), grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ MinTime: 15 * time.Second, PermitWithoutStream: true, diff --git a/pkg/gateway/http.go b/pkg/gateway/http.go index 9eebf297cd..eb4d5458c3 100644 --- a/pkg/gateway/http.go +++ b/pkg/gateway/http.go @@ -16,10 +16,13 @@ import ( "github.com/gin-contrib/pprof" "github.com/ttacon/chalk" + "google.golang.org/protobuf/reflect/protopath" + "google.golang.org/protobuf/reflect/protoreflect" "github.com/gin-gonic/gin" "github.com/prometheus/client_golang/prometheus" - "github.com/rancher/opni/pkg/config/v1beta1" + "github.com/rancher/opni/pkg/config/reactive" + configv1 "github.com/rancher/opni/pkg/config/v1" "github.com/rancher/opni/pkg/logger" "github.com/rancher/opni/pkg/plugins" "github.com/rancher/opni/pkg/plugins/apis/apiextensions" @@ -49,9 +52,9 @@ var ( type GatewayHTTPServer struct { router *gin.Engine - conf *v1beta1.GatewayConfigSpec + certs reactive.Value + mgr *configv1.GatewayConfigManager logger *slog.Logger - tlsConfig *tls.Config metricsRouter *gin.Engine metricsRegisterer prometheus.Registerer @@ -61,14 +64,23 @@ type GatewayHTTPServer struct { func NewHTTPServer( ctx context.Context, - cfg *v1beta1.GatewayConfigSpec, + mgr *configv1.GatewayConfigManager, lg *slog.Logger, pl plugins.LoaderInterface, -) *GatewayHTTPServer { +) (*GatewayHTTPServer, error) { lg = lg.WithGroup("http") router := gin.New() - router.SetTrustedProxies(cfg.TrustedProxies) + trustedProxies := mgr.Reactive(configv1.ProtoPath().Dashboard().TrustedProxies()) + + trustedProxies.WatchFunc(ctx, func(v protoreflect.Value) { + list := v.List() + var items []string + for i := 0; i < list.Len(); i++ { + items = append(items, list.Get(i).String()) + } + router.SetTrustedProxies(items) + }) router.Use( logger.GinLogger(lg), @@ -91,11 +103,7 @@ func NewHTTPServer( healthz.Store(http.StatusOK) })) - if cfg.Profiling.Path != "" { - pprof.Register(metricsRouter, cfg.Profiling.Path) - } else { - pprof.Register(metricsRouter) - } + pprof.Register(metricsRouter) metricsRouter.POST("/debug/profiles/:name/enable", func(c *gin.Context) { name := c.Param("name") @@ -129,26 +137,21 @@ func NewHTTPServer( fmt.Printf(chalk.Green.Color("%s profiling stopped\n"), name) }) - metricsHandler := NewMetricsEndpointHandler(cfg.Metrics) - metricsRouter.GET(cfg.Metrics.GetPath(), gin.WrapH(metricsHandler.Handler())) + metricsHandler := NewMetricsEndpointHandler() + metricsRouter.GET("/metrics", gin.WrapH(metricsHandler.Handler())) - tlsConfig, _, err := httpTLSConfig(cfg) - if err != nil { - lg.With( - logger.Err(err), - ).Error("failed to load serving cert bundle") - panic("failed to load serving cert bundle") - } + certs := mgr.Reactive(protopath.Path(configv1.ProtoPath().Certs())) srv := &GatewayHTTPServer{ router: router, - conf: cfg, + certs: certs, + mgr: mgr, logger: lg, - tlsConfig: tlsConfig, metricsRouter: metricsRouter, metricsRegisterer: metricsHandler.reg, reservedPrefixRoutes: []string{ - cfg.Metrics.GetPath(), "/healthz", + "/debug", + "/metrics", }, } @@ -177,60 +180,132 @@ func NewHTTPServer( })) pl.Hook(hooks.OnLoadM(func(p types.HTTPAPIExtensionPlugin, md meta.PluginMeta) { - ctx, ca := context.WithTimeout(ctx, 10*time.Second) - defer ca() - cfg, err := p.Configure(ctx, apiextensions.NewCertConfig(cfg.Certs)) - if err != nil { - lg.With( - "plugin", md.Module, - logger.Err(err), - ).Error("failed to configure routes") - return - } - srv.setupPluginRoutes(cfg, md) + certs.WatchFunc(ctx, func(v protoreflect.Value) { + ctx, ca := context.WithTimeout(ctx, 10*time.Second) + defer ca() + certs := v.Message().Interface().(*configv1.CertsSpec) + cfg, err := p.Configure(ctx, certs) + if err != nil { + lg.With( + "plugin", md.Module, + logger.Err(err), + ).Error("failed to configure routes") + return + } + srv.setupPluginRoutes(cfg, md, certs) + }) })) - return srv + return srv, nil } func (s *GatewayHTTPServer) ListenAndServe(ctx context.Context) error { lg := s.logger - listener, err := tls.Listen("tcp4", s.conf.HTTPListenAddress, s.tlsConfig) - if err != nil { - return err - } - - metricsListener, err := net.Listen("tcp4", s.conf.MetricsListenAddress) - if err != nil { - return err - } - - lg.With( - "api", listener.Addr().String(), - "metrics", metricsListener.Addr().String(), - ).Info("gateway HTTP server starting") - ctx, ca := context.WithCancelCause(ctx) + httpListenAddr := s.mgr.Reactive(configv1.ProtoPath().Server().HttpListenAddress()) + metricsListenAddr := s.mgr.Reactive(configv1.ProtoPath().Health().HttpListenAddress()) + e1 := lo.Async(func() error { - return util.ServeHandler(ctx, s.router.Handler(), listener) + var cancel context.CancelFunc + var done chan struct{} + reactive.Bind(ctx, func(v []protoreflect.Value) { + if cancel != nil { + cancel() + <-done + } + addr := v[0].String() + certs, err := v[1].Message().Interface().(*configv1.CertsSpec).AsTlsConfig(tls.NoClientCert) + if err != nil { + lg.With( + logger.Err(err), + ).Error("failed to configure TLS") + return + } + listener, err := tls.Listen("tcp4", addr, certs) + if err != nil { + lg.With( + "address", addr, + logger.Err(err), + ).Error("failed to start gateway HTTP server") + return + } + lg.With( + "address", listener.Addr().String(), + ).Info("gateway HTTP server starting") + + var serveContext context.Context + serveContext, cancel = context.WithCancel(ctx) + done = make(chan struct{}) + go func() { + defer close(done) + if err := util.ServeHandler(serveContext, s.router.Handler(), listener); err != nil { + lg.With(logger.Err(err)).Warn("gateway HTTP server exited with error") + } + }() + }, httpListenAddr, s.certs) + <-ctx.Done() + cancel() + <-done + return ctx.Err() }) e2 := lo.Async(func() error { - return util.ServeHandler(ctx, s.metricsRouter.Handler(), metricsListener) + var cancel context.CancelFunc + var done chan struct{} + metricsListenAddr.WatchFunc(ctx, func(v protoreflect.Value) { + if cancel != nil { + cancel() + <-done + } + addr := v.String() + listener, err := net.Listen("tcp4", addr) + if err != nil { + lg.With( + "address", addr, + logger.Err(err), + ).Error("failed to start metrics HTTP server") + return + } + lg.With( + "address", listener.Addr().String(), + ).Info("metrics HTTP server starting") + + var serveContext context.Context + serveContext, cancel = context.WithCancel(ctx) + done = make(chan struct{}) + go func() { + defer close(done) + if err := util.ServeHandler(serveContext, s.metricsRouter.Handler(), listener); err != nil { + lg.With(logger.Err(err)).Warn("metrics HTTP server exited with error") + } + }() + }) + <-ctx.Done() + cancel() + <-done + return ctx.Err() }) - return util.WaitAll(ctx, ca, e1, e2) + util.WaitAll(ctx, ca, e1, e2) + return context.Cause(ctx) } func (s *GatewayHTTPServer) setupPluginRoutes( cfg *apiextensions.HTTPAPIExtensionConfig, pluginMeta meta.PluginMeta, + certs *configv1.CertsSpec, ) { s.routesMu.Lock() defer s.routesMu.Unlock() - tlsConfig := s.tlsConfig.Clone() + tlsConfig, err := certs.AsTlsConfig(tls.NoClientCert) + if err != nil { + s.logger.With( + logger.Err(err), + ).Error("failed to configure TLS for plugin routes") + return + } sampledLogger := logger.New( logger.WithSampling(&slogsampling.ThresholdSamplingOption{Threshold: 1, Tick: logger.NoRepeatInterval, Rate: 0})).WithGroup("api") diff --git a/pkg/gateway/metrics.go b/pkg/gateway/metrics.go index 0941b55e76..653782768a 100644 --- a/pkg/gateway/metrics.go +++ b/pkg/gateway/metrics.go @@ -6,22 +6,19 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/rancher/opni/pkg/config/v1beta1" ) type MetricsEndpointHandler struct { reg *prometheus.Registry - cfg v1beta1.MetricsSpec } -func NewMetricsEndpointHandler(cfg v1beta1.MetricsSpec) *MetricsEndpointHandler { +func NewMetricsEndpointHandler() *MetricsEndpointHandler { reg := prometheus.NewRegistry() reg.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) reg.MustRegister(collectors.NewBuildInfoCollector()) reg.MustRegister(collectors.NewGoCollector()) return &MetricsEndpointHandler{ reg: reg, - cfg: cfg, } } diff --git a/pkg/gateway/ratelimit.go b/pkg/gateway/ratelimit.go index ab62fe434d..6ff496d926 100644 --- a/pkg/gateway/ratelimit.go +++ b/pkg/gateway/ratelimit.go @@ -1,9 +1,13 @@ package gateway import ( + "context" "fmt" "log/slog" + "sync/atomic" + "github.com/rancher/opni/pkg/config/reactive" + configv1 "github.com/rancher/opni/pkg/config/v1" tokenbucket "golang.org/x/time/rate" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -11,54 +15,30 @@ import ( ) type RatelimiterInterceptor struct { - tokenBucket *tokenbucket.Limiter + conf reactive.Reactive[*configv1.RateLimitingSpec] + tokenBucket atomic.Pointer[tokenbucket.Limiter] lg *slog.Logger } -type ratelimiterOptions struct { - rate float64 - burst int -} - -type RatelimiterOption func(*ratelimiterOptions) - -func (o *ratelimiterOptions) apply(opts ...RatelimiterOption) { - for _, opt := range opts { - opt(o) - } -} - -func WithRate(rate float64) RatelimiterOption { - return func(o *ratelimiterOptions) { - o.rate = rate - } -} - -func WithBurst(burst int) RatelimiterOption { - return func(o *ratelimiterOptions) { - o.burst = burst - } -} - -func NewRateLimiterInterceptor(lg *slog.Logger, opts ...RatelimiterOption) *RatelimiterInterceptor { - options := &ratelimiterOptions{ - rate: 10, - burst: 50, - } - options.apply(opts...) - - return &RatelimiterInterceptor{ - lg: lg, - tokenBucket: tokenbucket.NewLimiter( - tokenbucket.Limit(options.rate), - options.burst, - ), +func NewRateLimiterInterceptor(ctx context.Context, lg *slog.Logger, conf reactive.Reactive[*configv1.RateLimitingSpec]) *RatelimiterInterceptor { + r := &RatelimiterInterceptor{ + lg: lg, + conf: conf, } + conf.WatchFunc(ctx, func(value *configv1.RateLimitingSpec) { + r.lg.With("rate", value.GetRate(), "burst", value.GetBurst()).Info("updating ratelimit parameters") + r.tokenBucket.Store(tokenbucket.NewLimiter(tokenbucket.Limit(value.GetRate()), int(value.GetBurst()))) + }) + return r } func (r *RatelimiterInterceptor) allow() bool { - r.lg.Debug(fmt.Sprintf("ratelimit: %f available", r.tokenBucket.Tokens())) - return r.tokenBucket.Allow() + bucket := r.tokenBucket.Load() + if bucket == nil { + return true + } + r.lg.Debug(fmt.Sprintf("ratelimit: %f available", bucket.Tokens())) + return bucket.Allow() } func (r *RatelimiterInterceptor) StreamServerInterceptor() grpc.StreamServerInterceptor { diff --git a/pkg/machinery/upgrades.go b/pkg/machinery/upgrades.go index 6d2562b1e0..2a52590ce7 100644 --- a/pkg/machinery/upgrades.go +++ b/pkg/machinery/upgrades.go @@ -2,49 +2,46 @@ package machinery import ( "errors" + "log/slog" - "github.com/rancher/opni/pkg/config/v1beta1" + configv1 "github.com/rancher/opni/pkg/config/v1" "github.com/rancher/opni/pkg/oci" "github.com/rancher/opni/pkg/update" - "log/slog" ) -func ConfigurePluginUpgrader(cfg v1beta1.PluginUpgradeSpec, pluginDir string, lg *slog.Logger) (update.SyncHandler, error) { - switch cfg.Type { - case v1beta1.PluginUpgradeBinary: - builder := update.GetPluginSyncHandlerBuilder(cfg.Type) +func ConfigurePluginUpgrader(cfg *configv1.PluginUpgradesSpec, pluginDir string, lg *slog.Logger) (update.SyncHandler, error) { + switch cfg.GetDriver() { + case configv1.PluginUpgradesSpec_Binary: + builder := update.GetPluginSyncHandlerBuilder(cfg.GetDriver().String()) if builder == nil { return nil, errors.New("plugin provider not found") } return builder(pluginDir, lg) default: - builder := update.GetPluginSyncHandlerBuilder("noop") + builder := update.GetPluginSyncHandlerBuilder(configv1.PluginUpgradesSpec_Noop.String()) return builder() } } -func ConfigureAgentUpgrader(cfg *v1beta1.AgentUpgradeSpec, lg *slog.Logger) (update.SyncHandler, error) { - switch { - case cfg.Type == v1beta1.AgentUpgradeKubernetes: - builder := update.GetAgentSyncHandlerBuilder(cfg.Type) +func ConfigureAgentUpgrader(cfg *configv1.AgentUpgradesSpec, lg *slog.Logger) (update.SyncHandler, error) { + switch cfg.GetDriver() { + case configv1.AgentUpgradesSpec_Kubernetes: + builder := update.GetAgentSyncHandlerBuilder(cfg.GetDriver().String()) if cfg.Kubernetes != nil { - return builder(lg, cfg.Kubernetes.Namespace, cfg.Kubernetes.RepoOverride) + return builder(lg, cfg.GetKubernetes().GetNamespace(), cfg.Kubernetes.GetRepoOverride()) } return builder(lg) - case cfg.Type == v1beta1.AgentUpgradeNoop: - builder := update.GetAgentSyncHandlerBuilder(cfg.Type) + case configv1.AgentUpgradesSpec_Noop: + builder := update.GetAgentSyncHandlerBuilder(cfg.GetDriver().String()) return builder() default: - builder := update.GetAgentSyncHandlerBuilder("noop") + builder := update.GetAgentSyncHandlerBuilder(configv1.AgentUpgradesSpec_Kubernetes.String()) return builder() } } -func ConfigureOCIFetcher(providerType string, args ...any) (oci.Fetcher, error) { - if providerType == "" { - providerType = "noop" - } - builder := oci.GetFetcherBuilder(providerType) +func ConfigureOCIFetcher(providerType configv1.KubernetesAgentUpgradeSpec_ImageResolver, args ...any) (oci.Fetcher, error) { + builder := oci.GetFetcherBuilder(providerType.String()) if builder == nil { return nil, errors.New("oci provider not found") } diff --git a/pkg/management/apiextensions.go b/pkg/management/apiextensions.go index 5154ff418e..8945e0e061 100644 --- a/pkg/management/apiextensions.go +++ b/pkg/management/apiextensions.go @@ -170,11 +170,11 @@ func (m *Server) configureApiExtensionDirector(ctx context.Context, pl plugins.L } } -func (m *Server) configureManagementHttpApi(ctx context.Context, mux *runtime.ServeMux) error { +func (m *Server) configureManagementHttpApi(ctx context.Context, server *http.Server, addr string, mux *runtime.ServeMux) error { m.apiExtMu.RLock() defer m.apiExtMu.RUnlock() - cc, err := grpc.DialContext(ctx, m.config.GRPCListenAddress, + cc, err := grpc.DialContext(ctx, addr, grpc.WithBlock(), grpc.WithContextDialer(util.DialProtocol), grpc.WithTransportCredentials(insecure.NewCredentials()), @@ -193,22 +193,23 @@ func (m *Server) configureManagementHttpApi(ctx context.Context, mux *runtime.Se status := &health.ServingStatus{} // local server, which is always serving status.Set(healthpb.HealthCheckResponse_SERVING) - m.configureServiceStubHandlers(mux, stub, desc, rules, status) + m.configureServiceStubHandlers(server, mux, stub, desc, rules, status) return nil } -func (m *Server) configureHttpApiExtensions(mux *runtime.ServeMux) { +func (m *Server) configureHttpApiExtensions(server *http.Server, mux *runtime.ServeMux) { m.apiExtMu.RLock() defer m.apiExtMu.RUnlock() for _, ext := range m.apiExtensions { stub := grpcdynamic.NewStub(ext.clientConn) svcDesc := ext.serviceDesc httpRules := ext.httpRules - m.configureServiceStubHandlers(mux, stub, svcDesc, httpRules, ext.status) + m.configureServiceStubHandlers(server, mux, stub, svcDesc, httpRules, ext.status) } } func (m *Server) configureServiceStubHandlers( + server *http.Server, mux *runtime.ServeMux, stub grpcdynamic.Stub, svcDesc *desc.ServiceDescriptor, @@ -240,17 +241,17 @@ func (m *Server) configureServiceStubHandlers( qualifiedPath := fmt.Sprintf("/%s%s", svcDesc.GetName(), path) - if err := mux.HandlePath(method, qualifiedPath, newHandler(stub, svcDesc, mux, rule, svcStatus, path)); err != nil { + if err := mux.HandlePath(method, qualifiedPath, newHandler(server, stub, svcDesc, mux, rule, svcStatus, path)); err != nil { lg.With( logger.Err(err), "method", method, "path", qualifiedPath, ).Error("failed to configure http handler") } else { - lg.With( - "method", method, - "path", qualifiedPath, - ).Debug("configured http handler") + // lg.With( + // "method", method, + // "path", qualifiedPath, + // ).Debug("configured http handler") } } } @@ -282,6 +283,7 @@ func loadHttpRuleDescriptors(svc *desc.ServiceDescriptor) []*managementv1.HTTPRu } func newHandler( + server *http.Server, stub grpcdynamic.Stub, svcDesc *desc.ServiceDescriptor, mux *runtime.ServeMux, @@ -339,6 +341,7 @@ func newHandler( lg.With(logger.Err(err)).Error("failed to upgrade connection") return } + server.RegisterOnShutdown(func() { conn.Close() }) if err := handleWebsocketStream(ctx, conn, stub, methodDesc, lg); err != nil { lg.With(logger.Err(err)).Error("websocket stream error") diff --git a/pkg/management/config_test.go b/pkg/management/config_test.go deleted file mode 100644 index c76db23100..0000000000 --- a/pkg/management/config_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package management_test - -import ( - "context" - "encoding/json" - - "github.com/alecthomas/jsonschema" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - managementv1 "github.com/rancher/opni/pkg/apis/management/v1" - "github.com/rancher/opni/pkg/config" - "github.com/rancher/opni/pkg/config/meta" - "github.com/rancher/opni/pkg/config/v1beta1" - "github.com/rancher/opni/pkg/management" - "github.com/rancher/opni/pkg/plugins" - "google.golang.org/protobuf/types/known/emptypb" -) - -var _ = Describe("Management Config", Ordered, Label("unit", "slow"), func() { - var tv *testVars - var sampleObjects []meta.Object - var lifecycler config.Lifecycler - - BeforeAll(func() { - sampleObjects = []meta.Object{ - &v1beta1.GatewayConfig{ - TypeMeta: meta.TypeMeta{ - Kind: "GatewayConfig", - APIVersion: "v1beta1", - }, - Spec: v1beta1.GatewayConfigSpec{ - GRPCListenAddress: "foo", - HTTPListenAddress: "bar", - Management: v1beta1.ManagementSpec{ - GRPCListenAddress: "bar", - HTTPListenAddress: "baz", - }, - }, - }, - } - lifecycler = config.NewLifecycler(sampleObjects) - setupManagementServer(&tv, plugins.NoopLoader, management.WithLifecycler(lifecycler))() - }) - - It("should retrieve the current config", func() { - docs, err := tv.client.GetConfig(context.Background(), &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - objects := meta.ObjectList{} - for _, doc := range docs.Documents { - obj, err := config.LoadObject(doc.Json) - Expect(err).NotTo(HaveOccurred()) - objects = append(objects, obj) - } - ok := false - objects.Visit(func(obj *v1beta1.GatewayConfig, schema *jsonschema.Schema) { - ok = true - Expect(obj).To(Equal(sampleObjects[0])) - }) - Expect(ok).To(BeTrue()) - }) - It("should update the config", func() { - newObj := &v1beta1.GatewayConfig{ - TypeMeta: meta.TypeMeta{ - Kind: "GatewayConfig", - APIVersion: "v1beta1", - }, - Spec: v1beta1.GatewayConfigSpec{ - GRPCListenAddress: "foo2", - }, - } - doc, err := json.Marshal(newObj) - Expect(err).NotTo(HaveOccurred()) - reloadC, err := lifecycler.ReloadC() - Expect(err).NotTo(HaveOccurred()) - - go func() { - <-reloadC - }() - _, err = tv.client.UpdateConfig(context.Background(), &managementv1.UpdateConfigRequest{ - Documents: []*managementv1.ConfigDocument{ - { - Json: doc, - }, - }, - }) - Expect(err).NotTo(HaveOccurred()) - - docs, err := tv.client.GetConfig(context.Background(), &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - objects := meta.ObjectList{} - for _, doc := range docs.Documents { - obj, err := config.LoadObject(doc.Json) - Expect(err).NotTo(HaveOccurred()) - objects = append(objects, obj) - } - ok := false - objects.Visit(func(obj *v1beta1.GatewayConfig, schema *jsonschema.Schema) { - ok = true - Expect(obj).To(Equal(newObj)) - }) - Expect(ok).To(BeTrue()) - }) -}) diff --git a/pkg/management/management_suite_test.go b/pkg/management/management_suite_test.go index f7715fec7a..59aabba46e 100644 --- a/pkg/management/management_suite_test.go +++ b/pkg/management/management_suite_test.go @@ -10,17 +10,21 @@ import ( . "github.com/onsi/gomega" "github.com/prometheus/client_golang/prometheus" managementv1 "github.com/rancher/opni/pkg/apis/management/v1" - "github.com/rancher/opni/pkg/config/v1beta1" + configv1 "github.com/rancher/opni/pkg/config/v1" "github.com/rancher/opni/pkg/logger" "github.com/rancher/opni/pkg/management" "github.com/rancher/opni/pkg/plugins" "github.com/rancher/opni/pkg/plugins/hooks" "github.com/rancher/opni/pkg/storage" + "github.com/rancher/opni/pkg/storage/inmemory" "github.com/rancher/opni/pkg/test/freeport" mock_storage "github.com/rancher/opni/pkg/test/mock/storage" _ "github.com/rancher/opni/pkg/test/setup" "github.com/rancher/opni/pkg/test/testdata" "github.com/rancher/opni/pkg/test/testlog" + "github.com/rancher/opni/pkg/util" + "github.com/rancher/opni/pkg/util/flagutil" + "github.com/samber/lo" "go.uber.org/mock/gomock" "google.golang.org/grpc" ) @@ -66,10 +70,19 @@ func setupManagementServer(vars **testVars, pl plugins.LoaderInterface, opts ... ctx, ca := context.WithCancel(context.Background()) tv.storageBackend = mock_storage.NewTestStorageBackend(ctx, tv.ctrl) ports := freeport.GetFreePorts(2) - conf := &v1beta1.ManagementSpec{ - GRPCListenAddress: fmt.Sprintf("tcp://127.0.0.1:%d", ports[0]), - HTTPListenAddress: fmt.Sprintf("127.0.0.1:%d", ports[1]), - } + + defaultStore := inmemory.NewValueStore[*configv1.GatewayConfigSpec](util.ProtoClone) + activeStore := inmemory.NewValueStore[*configv1.GatewayConfigSpec](util.ProtoClone) + mgr := configv1.NewGatewayConfigManager(defaultStore, activeStore, flagutil.LoadDefaults) + mgr.SetConfiguration(ctx, &configv1.SetRequest{ + Spec: &configv1.GatewayConfigSpec{ + Management: &configv1.ManagementServerSpec{ + GrpcListenAddress: lo.ToPtr(fmt.Sprintf("tcp://127.0.0.1:%d", ports[0])), + HttpListenAddress: lo.ToPtr(fmt.Sprintf("127.0.0.1:%d", ports[1])), + }, + }, + }) + Expect(mgr.Start(ctx)).To(Succeed()) cert, err := tls.X509KeyPair(testdata.TestData("localhost.crt"), testdata.TestData("localhost.key")) Expect(err).NotTo(HaveOccurred()) cds := &testCoreDataSource{ @@ -78,7 +91,7 @@ func setupManagementServer(vars **testVars, pl plugins.LoaderInterface, opts ... Certificates: []tls.Certificate{cert}, }, } - server := management.NewServer(ctx, conf, cds, pl, opts...) + server := management.NewServer(ctx, cds, mgr, pl, opts...) tv.coreDataSource = cds tv.ifaces.collector = server pl.Hook(hooks.OnLoadingCompleted(func(int) { diff --git a/pkg/management/server.go b/pkg/management/server.go index 4b1d8d1fcd..f07a4dfe11 100644 --- a/pkg/management/server.go +++ b/pkg/management/server.go @@ -4,7 +4,6 @@ import ( "context" "crypto/tls" "crypto/x509" - "errors" "fmt" "net" "net/http" @@ -24,7 +23,8 @@ import ( "github.com/rancher/opni/pkg/caching" "github.com/rancher/opni/pkg/capabilities" "github.com/rancher/opni/pkg/config" - "github.com/rancher/opni/pkg/config/v1beta1" + "github.com/rancher/opni/pkg/config/reactive" + configv1 "github.com/rancher/opni/pkg/config/v1" "github.com/rancher/opni/pkg/health" "github.com/rancher/opni/pkg/logger" "github.com/rancher/opni/pkg/pkp" @@ -42,6 +42,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" + "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/types/known/emptypb" ) @@ -75,14 +76,14 @@ type Server struct { managementv1.UnsafeManagementServer managementv1.UnimplementedLocalPasswordServer managementServerOptions - config *v1beta1.ManagementSpec + mgr *configv1.GatewayConfigManager rbacManagerStore capabilities.RBACManagerStore logger *slog.Logger coreDataSource CoreDataSource - grpcServer *grpc.Server dashboardSettings *DashboardSettingsManager router *gin.Engine localAuth local.LocalAuthenticator + director StreamDirector apiExtMu sync.RWMutex apiExtensions []apiExtension @@ -104,12 +105,6 @@ func (o *managementServerOptions) apply(opts ...ManagementServerOption) { } } -func WithLifecycler(lc config.Lifecycler) ManagementServerOption { - return func(o *managementServerOptions) { - o.lifecycler = lc - } -} - func WithCapabilitiesDataSource(src CapabilitiesDataSource) ManagementServerOption { return func(o *managementServerOptions) { o.capabilitiesDataSource = src @@ -124,8 +119,8 @@ func WithHealthStatusDataSource(src HealthStatusDataSource) ManagementServerOpti func NewServer( ctx context.Context, - conf *v1beta1.ManagementSpec, cds CoreDataSource, + mgr *configv1.GatewayConfigManager, pluginLoader plugins.LoaderInterface, opts ...ManagementServerOption, ) *Server { @@ -135,7 +130,7 @@ func NewServer( m := &Server{ managementServerOptions: options, - config: conf, + mgr: mgr, logger: lg, coreDataSource: cds, rbacManagerStore: capabilities.NewRBACManagerStore(lg), @@ -145,30 +140,11 @@ func NewServer( }, router: gin.New(), } - - director := m.configureApiExtensionDirector(ctx, pluginLoader) - m.grpcServer = grpc.NewServer( - grpc.Creds(insecure.NewCredentials()), - grpc.UnknownServiceHandler(unknownServiceHandler(director)), - grpc.ChainStreamInterceptor(otelgrpc.StreamServerInterceptor()), - grpc.ChainUnaryInterceptor( - caching.NewClientGrpcTtlCacher().UnaryServerInterceptor(), - otelgrpc.UnaryServerInterceptor()), - ) - managementv1.RegisterManagementServer(m.grpcServer, m) - managementv1.RegisterLocalPasswordServer(m.grpcServer, m) - channelzservice.RegisterChannelzServiceToServer(m.grpcServer) + m.director = m.configureApiExtensionDirector(ctx, pluginLoader) pluginLoader.Hook(hooks.OnLoadM(func(sp types.SystemPlugin, md meta.PluginMeta) { + go sp.ServeConfigAPI(m.mgr) go sp.ServeManagementAPI(m) - go func() { - if err := sp.ServeAPIExtensions(m.config.GRPCListenAddress); err != nil { - lg.With( - "plugin", md.Module, - logger.Err(err), - ).Error("failed to serve plugin API extensions") - } - }() })) pluginLoader.Hook(hooks.OnLoadM(func(p types.CapabilityRBACPlugin, md meta.PluginMeta) { @@ -212,85 +188,184 @@ type managementApiServer interface { func (m *Server) ListenAndServe(ctx context.Context) error { ctx, ca := context.WithCancelCause(ctx) + channels := []<-chan error{ + // start grpc server + lo.Async(func() error { + err := m.listenAndServeGrpc(ctx) + if err != nil { + return fmt.Errorf("management grpc server exited with error: %w", err) + } + m.logger.Info("management grpc server stopped") + return nil + }), + // start http server + lo.Async(func() error { + err := m.listenAndServeHttp(ctx) + if err != nil { + return fmt.Errorf("management http server exited with error: %w", err) + } + m.logger.Info("management http server stopped") + return nil + }), + } - e1 := lo.Async(func() error { - err := m.listenAndServeGrpc(ctx) - if err != nil { - return fmt.Errorf("management grpc server exited with error: %w", err) - } - return nil - }) - - e2 := lo.Async(func() error { - err := m.listenAndServeHttp(ctx) - if err != nil { - return fmt.Errorf("management http server exited with error: %w", err) - } - return nil - }) - - return util.WaitAll(ctx, ca, e1, e2) + util.WaitAll(ctx, ca, channels...) + return context.Cause(ctx) } -func (m *Server) listenAndServeGrpc(ctx context.Context) error { - if m.config.GRPCListenAddress == "" { - return errors.New("GRPCListenAddress not configured") - } - lg := m.logger - listener, err := util.NewProtocolListener(m.config.GRPCListenAddress) - if err != nil { - return err - } - lg.With( - "address", listener.Addr().String(), - ).Info("management gRPC server starting") +// func (m *Server) ListenAndServe(ctx context.Context) error { +// var serveContext context.Context +// var cancel context.CancelCauseFunc +// var done chan struct{} + +// reactive.Message[*configv1.ManagementServerSpec](m.mgr.Reactive(protopath.Path(configv1.ProtoPath().Management()))). +// WatchFunc(ctx, func(conf *configv1.ManagementServerSpec) { +// if cancel != nil { +// m.logger.Info("configuration updated; reloading servers") +// cancel(errors.New("configuration updated")) +// m.logger.Debug("waiting for servers to stop...") +// <-done +// m.logger.Debug("servers stopped; reloading") +// } +// serveContext, cancel = context.WithCancelCause(ctx) +// done = make(chan struct{}) + +// e1 := lo.Async(func() error { +// err := m.listenAndServeGrpc(serveContext, conf) +// if err != nil { +// return fmt.Errorf("management grpc server exited with error: %w", err) +// } +// m.logger.Info("management grpc server stopped") +// return nil +// }) + +// e2 := lo.Async(func() error { +// err := m.listenAndServeHttp(serveContext, conf) +// if err != nil { +// return fmt.Errorf("management http server exited with error: %w", err) +// } +// m.logger.Info("management http server stopped") +// return nil +// }) + +// go func() { +// defer close(done) +// util.WaitAll(serveContext, cancel, e1, e2) +// }() +// }) +// <-ctx.Done() +// cancel(ctx.Err()) +// <-done +// return context.Cause(ctx) +// } - errC := lo.Async(func() error { - return m.grpcServer.Serve(listener) +func (m *Server) listenAndServeGrpc(ctx context.Context) error { + grpcListenAddr := m.mgr.Reactive(configv1.ProtoPath().Management().GrpcListenAddress()) + + var server *grpc.Server + var done chan struct{} + grpcListenAddr.WatchFunc(ctx, func(v protoreflect.Value) { + if server != nil { + server.Stop() + <-done + } + done = make(chan struct{}) + + server = grpc.NewServer( + grpc.Creds(insecure.NewCredentials()), + grpc.UnknownServiceHandler(unknownServiceHandler(m.director)), + grpc.ChainStreamInterceptor(otelgrpc.StreamServerInterceptor()), + grpc.ChainUnaryInterceptor( + caching.NewClientGrpcTtlCacher().UnaryServerInterceptor(), + otelgrpc.UnaryServerInterceptor()), + ) + managementv1.RegisterManagementServer(server, m) + configv1.RegisterGatewayConfigServer(server, m.mgr) + managementv1.RegisterLocalPasswordServer(server, m) + channelzservice.RegisterChannelzServiceToServer(server) + + addr := v.String() + listener, err := util.NewProtocolListener(addr) + if err != nil { + m.logger.With( + "address", addr, + logger.Err(err), + ).Error("failed to start management gRPC server") + return + } + m.logger.With( + "address", listener.Addr().String(), + ).Info("management gRPC server starting") + go func() { + defer close(done) + if err := server.Serve(listener); err != nil { + m.logger.With(logger.Err(err)).Warn("management gRPC server exited with error") + } else { + m.logger.Info("management gRPC server stopped") + } + }() }) - select { - case <-ctx.Done(): - m.grpcServer.Stop() - return ctx.Err() - case err := <-errC: - return err + <-ctx.Done() + if server != nil { + server.Stop() + <-done } + return ctx.Err() } func (m *Server) listenAndServeHttp(ctx context.Context) error { - if m.config.GRPCListenAddress == "" { - return errors.New("GRPCListenAddress not configured") - } - lg := m.logger - lg.With( - "address", m.config.HTTPListenAddress, - ).Info("management HTTP server starting") - gwmux := runtime.NewServeMux( - runtime.WithMarshalerOption("application/json", &LegacyJsonMarshaler{}), - runtime.WithMarshalerOption("application/octet-stream", &DynamicV1Marshaler{}), - runtime.WithMarshalerOption(runtime.MIMEWildcard, &DynamicV1Marshaler{}), - ) - - m.configureManagementHttpApi(ctx, gwmux) - m.configureHttpApiExtensions(gwmux) - m.router.Any("/*any", gin.WrapF(gwmux.ServeHTTP)) - server := &http.Server{ - Addr: m.config.HTTPListenAddress, - BaseContext: func(net.Listener) context.Context { - return ctx - }, - Handler: m.router.Handler(), - } - errC := lo.Async(func() error { - return server.ListenAndServe() - }) - select { - case <-ctx.Done(): - server.Close() - return ctx.Err() - case err := <-errC: - return err + httpListenAddr := m.mgr.Reactive(configv1.ProtoPath().Management().HttpListenAddress()) + grpcListenAddr := m.mgr.Reactive(configv1.ProtoPath().Management().GrpcListenAddress()) + + var server *http.Server + reactive.Bind(ctx, func(v []protoreflect.Value) { + if server != nil { + server.Shutdown(ctx) + } + httpAddr := v[0].String() + grpcAddr := v[1].String() + + listener, err := util.NewProtocolListener(httpAddr) + if err != nil { + m.logger.With( + "address", httpAddr, + logger.Err(err), + ).Error("failed to start management HTTP server") + return + } + m.logger.With( + "address", listener.Addr().String(), + ).Info("management HTTP server starting") + + gwmux := runtime.NewServeMux( + runtime.WithMarshalerOption("application/json", &LegacyJsonMarshaler{}), + runtime.WithMarshalerOption("application/octet-stream", &DynamicV1Marshaler{}), + runtime.WithMarshalerOption(runtime.MIMEWildcard, &DynamicV1Marshaler{}), + ) + + m.configureManagementHttpApi(ctx, server, grpcAddr, gwmux) + m.configureHttpApiExtensions(server, gwmux) + m.router.Any("/*any", gin.WrapF(gwmux.ServeHTTP)) + server = &http.Server{ + Addr: httpAddr, + BaseContext: func(net.Listener) context.Context { + return ctx + }, + Handler: m.router.Handler(), + } + go func() { + if err := server.Serve(listener); err != nil { + m.logger.With(logger.Err(err)).Warn("management HTTP server exited with error") + } else { + m.logger.Info("management HTTP server stopped") + } + }() + }, httpListenAddr, grpcListenAddr) + <-ctx.Done() + if server != nil { + server.Shutdown(ctx) } + return ctx.Err() } func (m *Server) CertsInfo(_ context.Context, _ *emptypb.Empty) (*managementv1.CertsInfoResponse, error) { @@ -349,7 +424,3 @@ func (m *Server) ListCapabilities(ctx context.Context, in *emptypb.Empty) (*mana Items: items, }, nil } - -func (m *Server) Server() *grpc.Server { - return m.grpcServer -} diff --git a/pkg/oci/oci.go b/pkg/oci/oci.go index cce9762ec9..c0c6835405 100644 --- a/pkg/oci/oci.go +++ b/pkg/oci/oci.go @@ -3,6 +3,7 @@ package oci import ( "context" "fmt" + "strings" "sync" "github.com/opencontainers/go-digest" @@ -128,11 +129,11 @@ var ( func RegisterFetcherBuilder[T ~string](name T, builder func(...any) (Fetcher, error)) { fetcherCacheMutex.Lock() defer fetcherCacheMutex.Unlock() - fetcherBuilderCache[string(name)] = builder + fetcherBuilderCache[strings.ToLower(string(name))] = builder } func GetFetcherBuilder[T ~string](name T) func(...any) (Fetcher, error) { fetcherCacheMutex.Lock() defer fetcherCacheMutex.Unlock() - return fetcherBuilderCache[string(name)] + return fetcherBuilderCache[strings.ToLower(string(name))] } diff --git a/pkg/plugins/apis/apiextensions/apiextensions.pb.go b/pkg/plugins/apis/apiextensions/apiextensions.pb.go index 8e9d320e7f..bf92e9967b 100644 --- a/pkg/plugins/apis/apiextensions/apiextensions.pb.go +++ b/pkg/plugins/apis/apiextensions/apiextensions.pb.go @@ -9,6 +9,7 @@ package apiextensions import ( totem "github.com/kralicky/totem" v1 "github.com/rancher/opni/pkg/apis/core/v1" + v11 "github.com/rancher/opni/pkg/config/v1" grpc_health_v1 "google.golang.org/grpc/health/grpc_health_v1" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" @@ -25,101 +26,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -type CertConfig struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Ca string `protobuf:"bytes,1,opt,name=ca,proto3" json:"ca,omitempty"` - CaData []byte `protobuf:"bytes,2,opt,name=caData,proto3" json:"caData,omitempty"` - Cert string `protobuf:"bytes,3,opt,name=cert,proto3" json:"cert,omitempty"` - CertData []byte `protobuf:"bytes,4,opt,name=certData,proto3" json:"certData,omitempty"` - Key string `protobuf:"bytes,5,opt,name=key,proto3" json:"key,omitempty"` - KeyData []byte `protobuf:"bytes,6,opt,name=keyData,proto3" json:"keyData,omitempty"` - Insecure bool `protobuf:"varint,7,opt,name=insecure,proto3" json:"insecure,omitempty"` -} - -func (x *CertConfig) Reset() { - *x = CertConfig{} - if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *CertConfig) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*CertConfig) ProtoMessage() {} - -func (x *CertConfig) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use CertConfig.ProtoReflect.Descriptor instead. -func (*CertConfig) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_rawDescGZIP(), []int{0} -} - -func (x *CertConfig) GetCa() string { - if x != nil { - return x.Ca - } - return "" -} - -func (x *CertConfig) GetCaData() []byte { - if x != nil { - return x.CaData - } - return nil -} - -func (x *CertConfig) GetCert() string { - if x != nil { - return x.Cert - } - return "" -} - -func (x *CertConfig) GetCertData() []byte { - if x != nil { - return x.CertData - } - return nil -} - -func (x *CertConfig) GetKey() string { - if x != nil { - return x.Key - } - return "" -} - -func (x *CertConfig) GetKeyData() []byte { - if x != nil { - return x.KeyData - } - return nil -} - -func (x *CertConfig) GetInsecure() bool { - if x != nil { - return x.Insecure - } - return false -} - type HTTPAPIExtensionConfig struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -132,7 +38,7 @@ type HTTPAPIExtensionConfig struct { func (x *HTTPAPIExtensionConfig) Reset() { *x = HTTPAPIExtensionConfig{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[1] + mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -145,7 +51,7 @@ func (x *HTTPAPIExtensionConfig) String() string { func (*HTTPAPIExtensionConfig) ProtoMessage() {} func (x *HTTPAPIExtensionConfig) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[1] + mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -158,7 +64,7 @@ func (x *HTTPAPIExtensionConfig) ProtoReflect() protoreflect.Message { // Deprecated: Use HTTPAPIExtensionConfig.ProtoReflect.Descriptor instead. func (*HTTPAPIExtensionConfig) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_rawDescGZIP(), []int{1} + return file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_rawDescGZIP(), []int{0} } func (x *HTTPAPIExtensionConfig) GetHttpAddr() string { @@ -186,7 +92,7 @@ type ServiceDescriptorProtoList struct { func (x *ServiceDescriptorProtoList) Reset() { *x = ServiceDescriptorProtoList{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[2] + mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -199,7 +105,7 @@ func (x *ServiceDescriptorProtoList) String() string { func (*ServiceDescriptorProtoList) ProtoMessage() {} func (x *ServiceDescriptorProtoList) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[2] + mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -212,7 +118,7 @@ func (x *ServiceDescriptorProtoList) ProtoReflect() protoreflect.Message { // Deprecated: Use ServiceDescriptorProtoList.ProtoReflect.Descriptor instead. func (*ServiceDescriptorProtoList) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_rawDescGZIP(), []int{2} + return file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_rawDescGZIP(), []int{1} } func (x *ServiceDescriptorProtoList) GetItems() []*descriptorpb.ServiceDescriptorProto { @@ -234,7 +140,7 @@ type RouteInfo struct { func (x *RouteInfo) Reset() { *x = RouteInfo{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[3] + mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -247,7 +153,7 @@ func (x *RouteInfo) String() string { func (*RouteInfo) ProtoMessage() {} func (x *RouteInfo) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[3] + mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -260,7 +166,7 @@ func (x *RouteInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use RouteInfo.ProtoReflect.Descriptor instead. func (*RouteInfo) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_rawDescGZIP(), []int{3} + return file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_rawDescGZIP(), []int{2} } func (x *RouteInfo) GetMethod() string { @@ -289,7 +195,7 @@ type AuthzRequest struct { func (x *AuthzRequest) Reset() { *x = AuthzRequest{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[4] + mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -302,7 +208,7 @@ func (x *AuthzRequest) String() string { func (*AuthzRequest) ProtoMessage() {} func (x *AuthzRequest) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[4] + mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -315,7 +221,7 @@ func (x *AuthzRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use AuthzRequest.ProtoReflect.Descriptor instead. func (*AuthzRequest) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_rawDescGZIP(), []int{4} + return file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_rawDescGZIP(), []int{3} } func (x *AuthzRequest) GetRoleList() *v1.ReferenceList { @@ -343,7 +249,7 @@ type AuthzResponse struct { func (x *AuthzResponse) Reset() { *x = AuthzResponse{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[5] + mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -356,7 +262,7 @@ func (x *AuthzResponse) String() string { func (*AuthzResponse) ProtoMessage() {} func (x *AuthzResponse) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[5] + mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -369,7 +275,7 @@ func (x *AuthzResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use AuthzResponse.ProtoReflect.Descriptor instead. func (*AuthzResponse) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_rawDescGZIP(), []int{5} + return file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_rawDescGZIP(), []int{4} } func (x *AuthzResponse) GetAuthorized() bool { @@ -391,7 +297,7 @@ type RequestDetails struct { func (x *RequestDetails) Reset() { *x = RequestDetails{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[6] + mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -404,7 +310,7 @@ func (x *RequestDetails) String() string { func (*RequestDetails) ProtoMessage() {} func (x *RequestDetails) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[6] + mi := &file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -417,7 +323,7 @@ func (x *RequestDetails) ProtoReflect() protoreflect.Message { // Deprecated: Use RequestDetails.ProtoReflect.Descriptor instead. func (*RequestDetails) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_rawDescGZIP(), []int{6} + return file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_rawDescGZIP(), []int{5} } func (x *RequestDetails) GetPath() string { @@ -445,100 +351,93 @@ var file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_pr 0x69, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x25, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6b, 0x72, 0x61, 0x6c, 0x69, 0x63, 0x6b, 0x79, 0x2f, 0x74, 0x6f, 0x74, 0x65, 0x6d, 0x2f, 0x74, 0x6f, 0x74, 0x65, 0x6d, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x1a, 0x1b, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2f, - 0x76, 0x31, 0x2f, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, - 0x20, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x33, - 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x61, 0x6e, 0x63, 0x68, - 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x6e, 0x69, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, - 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x22, 0xac, 0x01, 0x0a, 0x0a, 0x43, 0x65, 0x72, 0x74, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x12, 0x0e, 0x0a, 0x02, 0x63, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, - 0x63, 0x61, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x61, 0x44, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x06, 0x63, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x65, - 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x65, 0x72, 0x74, 0x12, 0x1a, - 0x0a, 0x08, 0x63, 0x65, 0x72, 0x74, 0x44, 0x61, 0x74, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x08, 0x63, 0x65, 0x72, 0x74, 0x44, 0x61, 0x74, 0x61, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x18, 0x0a, 0x07, - 0x6b, 0x65, 0x79, 0x44, 0x61, 0x74, 0x61, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x6b, - 0x65, 0x79, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1a, 0x0a, 0x08, 0x69, 0x6e, 0x73, 0x65, 0x63, 0x75, - 0x72, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x69, 0x6e, 0x73, 0x65, 0x63, 0x75, - 0x72, 0x65, 0x22, 0x66, 0x0a, 0x16, 0x48, 0x54, 0x54, 0x50, 0x41, 0x50, 0x49, 0x45, 0x78, 0x74, - 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, - 0x68, 0x74, 0x74, 0x70, 0x41, 0x64, 0x64, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, - 0x68, 0x74, 0x74, 0x70, 0x41, 0x64, 0x64, 0x72, 0x12, 0x30, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, - 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x61, 0x70, 0x69, 0x65, 0x78, - 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x49, 0x6e, - 0x66, 0x6f, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x5b, 0x0a, 0x1a, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, - 0x72, 0x6f, 0x74, 0x6f, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, - 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x37, 0x0a, 0x09, 0x52, 0x6f, 0x75, 0x74, 0x65, - 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x16, 0x0a, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x12, 0x0a, 0x04, - 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, - 0x22, 0x78, 0x0a, 0x0c, 0x41, 0x75, 0x74, 0x68, 0x7a, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x2f, 0x0a, 0x08, 0x72, 0x6f, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, - 0x6e, 0x63, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x08, 0x72, 0x6f, 0x6c, 0x65, 0x4c, 0x69, 0x73, - 0x74, 0x12, 0x37, 0x0a, 0x07, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, - 0x6e, 0x73, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, - 0x73, 0x52, 0x07, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x22, 0x2f, 0x0a, 0x0d, 0x41, 0x75, - 0x74, 0x68, 0x7a, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x61, - 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x0a, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x22, 0x38, 0x0a, 0x0e, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x12, 0x0a, - 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, - 0x68, 0x12, 0x12, 0x0a, 0x04, 0x76, 0x65, 0x72, 0x62, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x76, 0x65, 0x72, 0x62, 0x32, 0xe5, 0x02, 0x0a, 0x16, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x6d, 0x65, 0x6e, 0x74, 0x41, 0x50, 0x49, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x50, 0x0a, 0x0b, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x73, 0x12, - 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x29, 0x2e, 0x61, 0x70, 0x69, 0x65, 0x78, 0x74, - 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, - 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x4c, 0x69, - 0x73, 0x74, 0x12, 0x47, 0x0a, 0x0a, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, - 0x12, 0x1b, 0x2e, 0x61, 0x70, 0x69, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, - 0x2e, 0x41, 0x75, 0x74, 0x68, 0x7a, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, + 0x74, 0x6f, 0x1a, 0x3a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, + 0x61, 0x6e, 0x63, 0x68, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x6e, 0x69, 0x2f, 0x70, 0x6b, 0x67, 0x2f, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x76, 0x31, 0x2f, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, + 0x79, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, + 0x67, 0x72, 0x70, 0x63, 0x2f, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2f, 0x76, 0x31, 0x2f, 0x68, + 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x20, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x65, 0x73, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, + 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x33, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x72, 0x2f, 0x6f, + 0x70, 0x6e, 0x69, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x63, 0x6f, 0x72, + 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, + 0x66, 0x0a, 0x16, 0x48, 0x54, 0x54, 0x50, 0x41, 0x50, 0x49, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, + 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x68, 0x74, 0x74, + 0x70, 0x41, 0x64, 0x64, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x68, 0x74, 0x74, + 0x70, 0x41, 0x64, 0x64, 0x72, 0x12, 0x30, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, + 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x61, 0x70, 0x69, 0x65, 0x78, 0x74, 0x65, 0x6e, + 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, + 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x5b, 0x0a, 0x1a, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, + 0x6f, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x52, 0x05, 0x69, + 0x74, 0x65, 0x6d, 0x73, 0x22, 0x37, 0x0a, 0x09, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x49, 0x6e, 0x66, + 0x6f, 0x12, 0x16, 0x0a, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, + 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x78, 0x0a, + 0x0c, 0x41, 0x75, 0x74, 0x68, 0x7a, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2f, 0x0a, + 0x08, 0x72, 0x6f, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x13, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x4c, 0x69, 0x73, 0x74, 0x52, 0x08, 0x72, 0x6f, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x37, + 0x0a, 0x07, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x2e, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x07, + 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x22, 0x2f, 0x0a, 0x0d, 0x41, 0x75, 0x74, 0x68, 0x7a, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x61, 0x75, 0x74, 0x68, + 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x61, 0x75, + 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x22, 0x38, 0x0a, 0x0e, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, + 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, + 0x0a, 0x04, 0x76, 0x65, 0x72, 0x62, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x76, 0x65, + 0x72, 0x62, 0x32, 0xe5, 0x02, 0x0a, 0x16, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x41, 0x50, 0x49, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x50, 0x0a, + 0x0b, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x73, 0x12, 0x16, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x29, 0x2e, 0x61, 0x70, 0x69, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, + 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x73, 0x63, + 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x4c, 0x69, 0x73, 0x74, 0x12, + 0x47, 0x0a, 0x0a, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x12, 0x1b, 0x2e, 0x61, 0x70, 0x69, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x41, 0x75, - 0x74, 0x68, 0x7a, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x56, 0x0a, 0x0b, 0x43, - 0x68, 0x65, 0x63, 0x6b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x22, 0x2e, 0x67, 0x72, 0x70, - 0x63, 0x2e, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61, 0x6c, - 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, - 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x76, 0x31, 0x2e, - 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x0b, 0x57, 0x61, 0x74, 0x63, 0x68, 0x48, 0x65, 0x61, 0x6c, - 0x74, 0x68, 0x12, 0x22, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, - 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x68, 0x65, - 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, - 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x32, 0x61, 0x0a, - 0x10, 0x48, 0x54, 0x54, 0x50, 0x41, 0x50, 0x49, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, - 0x6e, 0x12, 0x4d, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x12, 0x19, - 0x2e, 0x61, 0x70, 0x69, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x43, - 0x65, 0x72, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x1a, 0x25, 0x2e, 0x61, 0x70, 0x69, 0x65, - 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x41, 0x50, - 0x49, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x32, 0x43, 0x0a, 0x12, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x41, 0x50, 0x49, 0x45, 0x78, 0x74, - 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x2d, 0x0a, 0x0f, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x12, 0x0a, 0x2e, 0x74, 0x6f, 0x74, 0x65, - 0x6d, 0x2e, 0x52, 0x50, 0x43, 0x1a, 0x0a, 0x2e, 0x74, 0x6f, 0x74, 0x65, 0x6d, 0x2e, 0x52, 0x50, - 0x43, 0x28, 0x01, 0x30, 0x01, 0x32, 0x67, 0x0a, 0x11, 0x55, 0x6e, 0x61, 0x72, 0x79, 0x41, 0x50, - 0x49, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x52, 0x0a, 0x0f, 0x55, 0x6e, - 0x61, 0x72, 0x79, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, + 0x74, 0x68, 0x7a, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x61, 0x70, 0x69, + 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x41, 0x75, 0x74, 0x68, 0x7a, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x56, 0x0a, 0x0b, 0x43, 0x68, 0x65, 0x63, + 0x6b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x22, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x68, + 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, + 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61, + 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x58, 0x0a, 0x0b, 0x57, 0x61, 0x74, 0x63, 0x68, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, + 0x22, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x76, 0x31, + 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x68, 0x65, 0x61, 0x6c, 0x74, + 0x68, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x32, 0x5c, 0x0a, 0x10, 0x48, 0x54, + 0x54, 0x50, 0x41, 0x50, 0x49, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x48, + 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x12, 0x14, 0x2e, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x65, 0x72, 0x74, 0x73, 0x53, 0x70, 0x65, + 0x63, 0x1a, 0x25, 0x2e, 0x61, 0x70, 0x69, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, + 0x73, 0x2e, 0x48, 0x54, 0x54, 0x50, 0x41, 0x50, 0x49, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, + 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x32, 0x43, 0x0a, 0x12, 0x53, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x41, 0x50, 0x49, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x2d, + 0x0a, 0x0f, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x12, 0x0a, 0x2e, 0x74, 0x6f, 0x74, 0x65, 0x6d, 0x2e, 0x52, 0x50, 0x43, 0x1a, 0x0a, 0x2e, + 0x74, 0x6f, 0x74, 0x65, 0x6d, 0x2e, 0x52, 0x50, 0x43, 0x28, 0x01, 0x30, 0x01, 0x32, 0x67, 0x0a, + 0x11, 0x55, 0x6e, 0x61, 0x72, 0x79, 0x41, 0x50, 0x49, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x52, 0x0a, 0x0f, 0x55, 0x6e, 0x61, 0x72, 0x79, 0x44, 0x65, 0x73, 0x63, 0x72, + 0x69, 0x70, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x27, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x27, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, - 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x42, 0x38, - 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x61, 0x6e, - 0x63, 0x68, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x6e, 0x69, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x6c, - 0x75, 0x67, 0x69, 0x6e, 0x73, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x61, 0x70, 0x69, 0x65, 0x78, - 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, + 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x42, 0x38, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x6e, + 0x69, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x73, 0x2f, 0x61, 0x70, + 0x69, 0x73, 0x2f, 0x61, 0x70, 0x69, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -553,41 +452,41 @@ func file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_p return file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_rawDescData } -var file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes = make([]protoimpl.MessageInfo, 7) +var file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes = make([]protoimpl.MessageInfo, 6) var file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_goTypes = []interface{}{ - (*CertConfig)(nil), // 0: apiextensions.CertConfig - (*HTTPAPIExtensionConfig)(nil), // 1: apiextensions.HTTPAPIExtensionConfig - (*ServiceDescriptorProtoList)(nil), // 2: apiextensions.ServiceDescriptorProtoList - (*RouteInfo)(nil), // 3: apiextensions.RouteInfo - (*AuthzRequest)(nil), // 4: apiextensions.AuthzRequest - (*AuthzResponse)(nil), // 5: apiextensions.AuthzResponse - (*RequestDetails)(nil), // 6: apiextensions.RequestDetails - (*descriptorpb.ServiceDescriptorProto)(nil), // 7: google.protobuf.ServiceDescriptorProto - (*v1.ReferenceList)(nil), // 8: core.ReferenceList - (*emptypb.Empty)(nil), // 9: google.protobuf.Empty - (*grpc_health_v1.HealthCheckRequest)(nil), // 10: grpc.health.v1.HealthCheckRequest + (*HTTPAPIExtensionConfig)(nil), // 0: apiextensions.HTTPAPIExtensionConfig + (*ServiceDescriptorProtoList)(nil), // 1: apiextensions.ServiceDescriptorProtoList + (*RouteInfo)(nil), // 2: apiextensions.RouteInfo + (*AuthzRequest)(nil), // 3: apiextensions.AuthzRequest + (*AuthzResponse)(nil), // 4: apiextensions.AuthzResponse + (*RequestDetails)(nil), // 5: apiextensions.RequestDetails + (*descriptorpb.ServiceDescriptorProto)(nil), // 6: google.protobuf.ServiceDescriptorProto + (*v1.ReferenceList)(nil), // 7: core.ReferenceList + (*emptypb.Empty)(nil), // 8: google.protobuf.Empty + (*grpc_health_v1.HealthCheckRequest)(nil), // 9: grpc.health.v1.HealthCheckRequest + (*v11.CertsSpec)(nil), // 10: config.v1.CertsSpec (*totem.RPC)(nil), // 11: totem.RPC (*grpc_health_v1.HealthCheckResponse)(nil), // 12: grpc.health.v1.HealthCheckResponse } var file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_depIdxs = []int32{ - 3, // 0: apiextensions.HTTPAPIExtensionConfig.routes:type_name -> apiextensions.RouteInfo - 7, // 1: apiextensions.ServiceDescriptorProtoList.items:type_name -> google.protobuf.ServiceDescriptorProto - 8, // 2: apiextensions.AuthzRequest.roleList:type_name -> core.ReferenceList - 6, // 3: apiextensions.AuthzRequest.details:type_name -> apiextensions.RequestDetails - 9, // 4: apiextensions.ManagementAPIExtension.Descriptors:input_type -> google.protobuf.Empty - 4, // 5: apiextensions.ManagementAPIExtension.Authorized:input_type -> apiextensions.AuthzRequest - 10, // 6: apiextensions.ManagementAPIExtension.CheckHealth:input_type -> grpc.health.v1.HealthCheckRequest - 10, // 7: apiextensions.ManagementAPIExtension.WatchHealth:input_type -> grpc.health.v1.HealthCheckRequest - 0, // 8: apiextensions.HTTPAPIExtension.Configure:input_type -> apiextensions.CertConfig + 2, // 0: apiextensions.HTTPAPIExtensionConfig.routes:type_name -> apiextensions.RouteInfo + 6, // 1: apiextensions.ServiceDescriptorProtoList.items:type_name -> google.protobuf.ServiceDescriptorProto + 7, // 2: apiextensions.AuthzRequest.roleList:type_name -> core.ReferenceList + 5, // 3: apiextensions.AuthzRequest.details:type_name -> apiextensions.RequestDetails + 8, // 4: apiextensions.ManagementAPIExtension.Descriptors:input_type -> google.protobuf.Empty + 3, // 5: apiextensions.ManagementAPIExtension.Authorized:input_type -> apiextensions.AuthzRequest + 9, // 6: apiextensions.ManagementAPIExtension.CheckHealth:input_type -> grpc.health.v1.HealthCheckRequest + 9, // 7: apiextensions.ManagementAPIExtension.WatchHealth:input_type -> grpc.health.v1.HealthCheckRequest + 10, // 8: apiextensions.HTTPAPIExtension.Configure:input_type -> config.v1.CertsSpec 11, // 9: apiextensions.StreamAPIExtension.ConnectInternal:input_type -> totem.RPC - 9, // 10: apiextensions.UnaryAPIExtension.UnaryDescriptor:input_type -> google.protobuf.Empty - 2, // 11: apiextensions.ManagementAPIExtension.Descriptors:output_type -> apiextensions.ServiceDescriptorProtoList - 5, // 12: apiextensions.ManagementAPIExtension.Authorized:output_type -> apiextensions.AuthzResponse + 8, // 10: apiextensions.UnaryAPIExtension.UnaryDescriptor:input_type -> google.protobuf.Empty + 1, // 11: apiextensions.ManagementAPIExtension.Descriptors:output_type -> apiextensions.ServiceDescriptorProtoList + 4, // 12: apiextensions.ManagementAPIExtension.Authorized:output_type -> apiextensions.AuthzResponse 12, // 13: apiextensions.ManagementAPIExtension.CheckHealth:output_type -> grpc.health.v1.HealthCheckResponse 12, // 14: apiextensions.ManagementAPIExtension.WatchHealth:output_type -> grpc.health.v1.HealthCheckResponse - 1, // 15: apiextensions.HTTPAPIExtension.Configure:output_type -> apiextensions.HTTPAPIExtensionConfig + 0, // 15: apiextensions.HTTPAPIExtension.Configure:output_type -> apiextensions.HTTPAPIExtensionConfig 11, // 16: apiextensions.StreamAPIExtension.ConnectInternal:output_type -> totem.RPC - 7, // 17: apiextensions.UnaryAPIExtension.UnaryDescriptor:output_type -> google.protobuf.ServiceDescriptorProto + 6, // 17: apiextensions.UnaryAPIExtension.UnaryDescriptor:output_type -> google.protobuf.ServiceDescriptorProto 11, // [11:18] is the sub-list for method output_type 4, // [4:11] is the sub-list for method input_type 4, // [4:4] is the sub-list for extension type_name @@ -602,18 +501,6 @@ func file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_p } if !protoimpl.UnsafeEnabled { file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CertConfig); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*HTTPAPIExtensionConfig); i { case 0: return &v.state @@ -625,7 +512,7 @@ func file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_p return nil } } - file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ServiceDescriptorProtoList); i { case 0: return &v.state @@ -637,7 +524,7 @@ func file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_p return nil } } - file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*RouteInfo); i { case 0: return &v.state @@ -649,7 +536,7 @@ func file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_p return nil } } - file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*AuthzRequest); i { case 0: return &v.state @@ -661,7 +548,7 @@ func file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_p return nil } } - file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*AuthzResponse); i { case 0: return &v.state @@ -673,7 +560,7 @@ func file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_p return nil } } - file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*RequestDetails); i { case 0: return &v.state @@ -692,7 +579,7 @@ func file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_p GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_github_com_rancher_opni_pkg_plugins_apis_apiextensions_apiextensions_proto_rawDesc, NumEnums: 0, - NumMessages: 7, + NumMessages: 6, NumExtensions: 0, NumServices: 4, }, diff --git a/pkg/plugins/apis/apiextensions/apiextensions.proto b/pkg/plugins/apis/apiextensions/apiextensions.proto index e649904ce8..c9ed97cc8e 100644 --- a/pkg/plugins/apis/apiextensions/apiextensions.proto +++ b/pkg/plugins/apis/apiextensions/apiextensions.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package apiextensions; import "github.com/kralicky/totem/totem.proto"; +import "github.com/rancher/opni/pkg/config/v1/gateway_config.proto"; import "google.golang.org/grpc/health/grpc_health_v1/health.proto"; import "google/protobuf/descriptor.proto"; import "google/protobuf/empty.proto"; @@ -19,7 +20,7 @@ service ManagementAPIExtension { } service HTTPAPIExtension { - rpc Configure(CertConfig) returns (HTTPAPIExtensionConfig); + rpc Configure(config.v1.CertsSpec) returns (HTTPAPIExtensionConfig); } service StreamAPIExtension { @@ -30,16 +31,6 @@ service UnaryAPIExtension { rpc UnaryDescriptor(google.protobuf.Empty) returns (google.protobuf.ServiceDescriptorProto); } -message CertConfig { - string ca = 1; - bytes caData = 2; - string cert = 3; - bytes certData = 4; - string key = 5; - bytes keyData = 6; - bool insecure = 7; -} - message HTTPAPIExtensionConfig { string httpAddr = 1; repeated RouteInfo routes = 2; diff --git a/pkg/plugins/apis/apiextensions/apiextensions_grpc.pb.go b/pkg/plugins/apis/apiextensions/apiextensions_grpc.pb.go index c270dc524f..cd003e5417 100644 --- a/pkg/plugins/apis/apiextensions/apiextensions_grpc.pb.go +++ b/pkg/plugins/apis/apiextensions/apiextensions_grpc.pb.go @@ -9,6 +9,7 @@ package apiextensions import ( context "context" totem "github.com/kralicky/totem" + v1 "github.com/rancher/opni/pkg/config/v1" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" grpc_health_v1 "google.golang.org/grpc/health/grpc_health_v1" @@ -257,7 +258,7 @@ const ( // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type HTTPAPIExtensionClient interface { - Configure(ctx context.Context, in *CertConfig, opts ...grpc.CallOption) (*HTTPAPIExtensionConfig, error) + Configure(ctx context.Context, in *v1.CertsSpec, opts ...grpc.CallOption) (*HTTPAPIExtensionConfig, error) } type hTTPAPIExtensionClient struct { @@ -268,7 +269,7 @@ func NewHTTPAPIExtensionClient(cc grpc.ClientConnInterface) HTTPAPIExtensionClie return &hTTPAPIExtensionClient{cc} } -func (c *hTTPAPIExtensionClient) Configure(ctx context.Context, in *CertConfig, opts ...grpc.CallOption) (*HTTPAPIExtensionConfig, error) { +func (c *hTTPAPIExtensionClient) Configure(ctx context.Context, in *v1.CertsSpec, opts ...grpc.CallOption) (*HTTPAPIExtensionConfig, error) { out := new(HTTPAPIExtensionConfig) err := c.cc.Invoke(ctx, HTTPAPIExtension_Configure_FullMethodName, in, out, opts...) if err != nil { @@ -281,14 +282,14 @@ func (c *hTTPAPIExtensionClient) Configure(ctx context.Context, in *CertConfig, // All implementations should embed UnimplementedHTTPAPIExtensionServer // for forward compatibility type HTTPAPIExtensionServer interface { - Configure(context.Context, *CertConfig) (*HTTPAPIExtensionConfig, error) + Configure(context.Context, *v1.CertsSpec) (*HTTPAPIExtensionConfig, error) } // UnimplementedHTTPAPIExtensionServer should be embedded to have forward compatible implementations. type UnimplementedHTTPAPIExtensionServer struct { } -func (UnimplementedHTTPAPIExtensionServer) Configure(context.Context, *CertConfig) (*HTTPAPIExtensionConfig, error) { +func (UnimplementedHTTPAPIExtensionServer) Configure(context.Context, *v1.CertsSpec) (*HTTPAPIExtensionConfig, error) { return nil, status.Errorf(codes.Unimplemented, "method Configure not implemented") } @@ -304,7 +305,7 @@ func RegisterHTTPAPIExtensionServer(s grpc.ServiceRegistrar, srv HTTPAPIExtensio } func _HTTPAPIExtension_Configure_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(CertConfig) + in := new(v1.CertsSpec) if err := dec(in); err != nil { return nil, err } @@ -316,7 +317,7 @@ func _HTTPAPIExtension_Configure_Handler(srv interface{}, ctx context.Context, d FullMethod: HTTPAPIExtension_Configure_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(HTTPAPIExtensionServer).Configure(ctx, req.(*CertConfig)) + return srv.(HTTPAPIExtensionServer).Configure(ctx, req.(*v1.CertsSpec)) } return interceptor(ctx, in, info, handler) } diff --git a/pkg/plugins/apis/apiextensions/http/plugin.go b/pkg/plugins/apis/apiextensions/http/plugin.go index ba9652b561..c039899384 100644 --- a/pkg/plugins/apis/apiextensions/http/plugin.go +++ b/pkg/plugins/apis/apiextensions/http/plugin.go @@ -10,6 +10,7 @@ import ( "go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin" "google.golang.org/grpc" + configv1 "github.com/rancher/opni/pkg/config/v1" "github.com/rancher/opni/pkg/plugins" "github.com/rancher/opni/pkg/plugins/apis/apiextensions" ) @@ -46,17 +47,17 @@ func (p *httpApiExtensionPlugin) GRPCServer( func (p *httpApiExtensionPlugin) Configure( _ context.Context, - certCfg *apiextensions.CertConfig, + certCfg *configv1.CertsSpec, ) (*apiextensions.HTTPAPIExtensionConfig, error) { var listener net.Listener var err error - if certCfg.Insecure { + if certCfg == nil { listener, err = net.Listen("tcp", "127.0.0.1:0") if err != nil { return nil, err } } else { - tlsCfg, err := certCfg.TLSConfig() + tlsCfg, err := certCfg.AsTlsConfig(tls.NoClientCert) if err != nil { return nil, err } diff --git a/pkg/plugins/apis/apiextensions/tls.go b/pkg/plugins/apis/apiextensions/tls.go deleted file mode 100644 index fa07047c64..0000000000 --- a/pkg/plugins/apis/apiextensions/tls.go +++ /dev/null @@ -1,59 +0,0 @@ -package apiextensions - -import ( - "crypto/tls" - - "github.com/rancher/opni/pkg/config/v1beta1" - "github.com/rancher/opni/pkg/util" - "github.com/samber/lo" -) - -func (tc *CertConfig) TLSConfig() (*tls.Config, error) { - certCfg := v1beta1.CertsSpec{} - if tc.Ca != "" { - certCfg.CACert = &tc.Ca - } - if tc.CaData != nil { - certCfg.CACertData = tc.CaData - } - if tc.Cert != "" { - certCfg.ServingCert = &tc.Cert - } - if tc.CertData != nil { - certCfg.ServingCertData = tc.CertData - } - if tc.Key != "" { - certCfg.ServingKey = &tc.Key - } - if tc.KeyData != nil { - certCfg.ServingKeyData = tc.KeyData - } - bundle, caPool, err := util.LoadServingCertBundle(certCfg) - if err != nil { - return nil, err - } - return &tls.Config{ - MinVersion: tls.VersionTLS12, - RootCAs: caPool, - ClientAuth: tls.RequireAndVerifyClientCert, - ClientCAs: caPool, - Certificates: []tls.Certificate{*bundle}, - }, nil -} - -func NewCertConfig(certs v1beta1.CertsSpec) *CertConfig { - return &CertConfig{ - Ca: lo.FromPtr(certs.CACert), - CaData: certs.CACertData, - Cert: lo.FromPtr(certs.ServingCert), - CertData: certs.ServingCertData, - Key: lo.FromPtr(certs.ServingKey), - KeyData: certs.ServingKeyData, - } -} - -func NewInsecureCertConfig() *CertConfig { - return &CertConfig{ - Insecure: true, - } -} diff --git a/pkg/plugins/apis/system/plugin.go b/pkg/plugins/apis/system/plugin.go index 4dac3747da..2f3f0a1610 100644 --- a/pkg/plugins/apis/system/plugin.go +++ b/pkg/plugins/apis/system/plugin.go @@ -8,6 +8,7 @@ import ( "github.com/hashicorp/go-plugin" managementv1 "github.com/rancher/opni/pkg/apis/management/v1" "github.com/rancher/opni/pkg/caching" + configv1 "github.com/rancher/opni/pkg/config/v1" "github.com/rancher/opni/pkg/plugins" "github.com/rancher/opni/pkg/storage" "github.com/rancher/opni/pkg/util" @@ -20,6 +21,7 @@ import ( ) type SystemPluginClient interface { + UseConfigAPI(configv1.GatewayConfigClient) UseManagementAPI(managementv1.ManagementClient) UseCachingProvider(caching.CachingProvider[proto.Message]) UseKeyValueStore(KeyValueStoreClient) @@ -30,6 +32,7 @@ type SystemPluginClient interface { // UnimplementedSystemPluginClient must be embedded to have forward compatible implementations. type UnimplementedSystemPluginClient struct{} +func (UnimplementedSystemPluginClient) UseConfigAPI(configv1.GatewayConfigClient) {} func (UnimplementedSystemPluginClient) UseManagementAPI(managementv1.ManagementClient) {} func (UnimplementedSystemPluginClient) UseKeyValueStore(KeyValueStoreClient) {} func (UnimplementedSystemPluginClient) UseAPIExtensions(ExtensionClientInterface) {} @@ -37,6 +40,7 @@ func (UnimplementedSystemPluginClient) UseCachingProvider(caching.CachingProvide func (UnimplementedSystemPluginClient) mustEmbedUnimplementedSystemPluginClient() {} type SystemPluginServer interface { + ServeConfigAPI(configv1.GatewayConfigServer) ServeManagementAPI(server managementv1.ManagementServer) ServeKeyValueStore(namespace string, backend storage.Backend) ServeAPIExtensions(dialAddress string) error @@ -83,6 +87,20 @@ type systemPluginClientImpl struct { cache caching.GrpcCachingInterceptor } +func (c *systemPluginClientImpl) UseConfigAPI(_ context.Context, in *BrokerID) (*emptypb.Empty, error) { + cc, err := c.broker.Dial(in.Id) + if err != nil { + return nil, err + } + defer cc.Close() + client := configv1.NewGatewayConfigClient(cc) + c.client.UseConfigAPI(client) + if err := c.startAPIExtensionClientHandler(client); err != nil { + return nil, err + } + return &emptypb.Empty{}, nil +} + func (c *systemPluginClientImpl) UseManagementAPI(_ context.Context, in *BrokerID) (*emptypb.Empty, error) { cc, err := c.broker.Dial( in.Id, @@ -109,34 +127,51 @@ func (c *systemPluginClientImpl) UseKeyValueStore(_ context.Context, in *BrokerI return &emptypb.Empty{}, nil } -func (c *systemPluginClientImpl) UseAPIExtensions(ctx context.Context, addr *DialAddress) (*emptypb.Empty, error) { - dialOpts := []grpc.DialOption{ - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithBlock(), - grpc.WithContextDialer(util.DialProtocol), - grpc.WithConnectParams(grpc.ConnectParams{ - Backoff: backoff.Config{ - BaseDelay: 1 * time.Millisecond, - Multiplier: 2, - Jitter: 0.2, - MaxDelay: 1 * time.Second, - }, - }), - grpc.WithChainUnaryInterceptor( - c.cache.UnaryClientInterceptor(), - ), - } - cc, err := grpc.DialContext(ctx, addr.Value, - dialOpts..., - ) +func (c *systemPluginClientImpl) startAPIExtensionClientHandler(client configv1.GatewayConfigClient) error { + ctx := context.TODO() + stream, err := client.WatchReactive(ctx, &configv1.ReactiveWatchRequest{ + Paths: []string{ + configv1.ProtoPath().Management().GrpcListenAddress().String(), + }, + }) if err != nil { - return nil, err + return err } - defer cc.Close() - c.client.UseAPIExtensions(&apiExtensionInterfaceImpl{ - managementClientConn: cc, - }) - return &emptypb.Empty{}, nil + go func() { + for { + events, err := stream.Recv() + if err != nil { + return + } + addr := events.Items[0].Value.ToValue().String() + + dialOpts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithBlock(), + grpc.WithContextDialer(util.DialProtocol), + grpc.WithConnectParams(grpc.ConnectParams{ + Backoff: backoff.Config{ + BaseDelay: 1 * time.Millisecond, + Multiplier: 2, + Jitter: 0.2, + MaxDelay: 1 * time.Second, + }, + }), + grpc.WithChainUnaryInterceptor( + c.cache.UnaryClientInterceptor(), + ), + } + cc, err := grpc.DialContext(ctx, addr, dialOpts...) + if err == nil { + c.client.UseAPIExtensions(&apiExtensionInterfaceImpl{ + managementClientConn: cc, + }) + cc.Close() + } + } + }() + + return nil } func (c *systemPluginClientImpl) UseCachingProvider(_ context.Context, _ *emptypb.Empty) (*emptypb.Empty, error) { @@ -167,6 +202,19 @@ type systemPluginHandler struct { client SystemClient } +func (s *systemPluginHandler) ServeConfigAPI(api configv1.GatewayConfigServer) { + s.serveSystemApi( + func(srv *grpc.Server) { + configv1.RegisterGatewayConfigServer(srv, api) + }, + func(id uint32) { + s.client.UseConfigAPI(s.ctx, &BrokerID{ + Id: id, + }) + }, + ) +} + func (s *systemPluginHandler) ServeManagementAPI(api managementv1.ManagementServer) { s.serveSystemApi( func(srv *grpc.Server) { @@ -200,13 +248,6 @@ func (s *systemPluginHandler) ServeKeyValueStore(namespace string, backend stora ) } -func (s *systemPluginHandler) ServeAPIExtensions(dialAddr string) error { - _, err := s.client.UseAPIExtensions(s.ctx, &DialAddress{ - Value: dialAddr, - }) - return err -} - func (s *systemPluginHandler) ServeCachingProvider() { s.client.UseCachingProvider(s.ctx, &emptypb.Empty{}) } diff --git a/pkg/plugins/apis/system/system.pb.go b/pkg/plugins/apis/system/system.pb.go index d6f3d6c903..69de480e19 100644 --- a/pkg/plugins/apis/system/system.pb.go +++ b/pkg/plugins/apis/system/system.pb.go @@ -1139,52 +1139,52 @@ var file_github_com_rancher_opni_pkg_plugins_apis_system_system_proto_rawDesc = 0x76, 0x65, 0x6e, 0x74, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, - 0x10, 0x02, 0x32, 0x8b, 0x02, 0x0a, 0x06, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x12, 0x3c, 0x0a, - 0x10, 0x55, 0x73, 0x65, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x41, 0x50, - 0x49, 0x12, 0x10, 0x2e, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x42, 0x72, 0x6f, 0x6b, 0x65, - 0x72, 0x49, 0x44, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x3c, 0x0a, 0x10, 0x55, - 0x73, 0x65, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, - 0x10, 0x2e, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x49, - 0x44, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x3f, 0x0a, 0x10, 0x55, 0x73, 0x65, - 0x41, 0x50, 0x49, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x13, 0x2e, - 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x44, 0x69, 0x61, 0x6c, 0x41, 0x64, 0x64, 0x72, 0x65, - 0x73, 0x73, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x44, 0x0a, 0x12, 0x55, 0x73, - 0x65, 0x43, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x67, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, - 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, - 0x32, 0x90, 0x03, 0x0a, 0x0d, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x53, 0x74, 0x6f, - 0x72, 0x65, 0x12, 0x2e, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x12, 0x2e, 0x73, 0x79, 0x73, 0x74, - 0x65, 0x6d, 0x2e, 0x50, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, - 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x50, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x12, 0x2e, 0x73, 0x79, 0x73, 0x74, - 0x65, 0x6d, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, - 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x36, 0x0a, 0x05, 0x57, 0x61, 0x74, 0x63, 0x68, 0x12, 0x14, 0x2e, 0x73, 0x79, - 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x57, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x15, 0x2e, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x57, 0x61, 0x74, 0x63, 0x68, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x37, 0x0a, 0x06, 0x44, 0x65, - 0x6c, 0x65, 0x74, 0x65, 0x12, 0x15, 0x2e, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x44, 0x65, - 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x73, 0x79, - 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x08, 0x4c, 0x69, 0x73, 0x74, 0x4b, 0x65, 0x79, 0x73, 0x12, - 0x17, 0x2e, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4b, 0x65, 0x79, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x73, 0x79, 0x73, 0x74, 0x65, - 0x6d, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4b, 0x65, 0x79, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x3a, 0x0a, 0x07, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x16, 0x2e, - 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x48, - 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, - 0x0a, 0x04, 0x4c, 0x6f, 0x63, 0x6b, 0x12, 0x13, 0x2e, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, - 0x4c, 0x6f, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x73, 0x79, - 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x4c, 0x6f, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x30, 0x01, 0x42, 0x31, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x6e, 0x69, 0x2f, 0x70, - 0x6b, 0x67, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x73, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, - 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x10, 0x02, 0x32, 0x84, 0x02, 0x0a, 0x06, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x12, 0x38, 0x0a, + 0x0c, 0x55, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x41, 0x50, 0x49, 0x12, 0x10, 0x2e, + 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x49, 0x44, 0x1a, + 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x3c, 0x0a, 0x10, 0x55, 0x73, 0x65, 0x4d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x41, 0x50, 0x49, 0x12, 0x10, 0x2e, 0x73, 0x79, + 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x49, 0x44, 0x1a, 0x16, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x3c, 0x0a, 0x10, 0x55, 0x73, 0x65, 0x4b, 0x65, 0x79, 0x56, + 0x61, 0x6c, 0x75, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x10, 0x2e, 0x73, 0x79, 0x73, 0x74, + 0x65, 0x6d, 0x2e, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x49, 0x44, 0x1a, 0x16, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x12, 0x44, 0x0a, 0x12, 0x55, 0x73, 0x65, 0x43, 0x61, 0x63, 0x68, 0x69, 0x6e, + 0x67, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, + 0x79, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x32, 0x90, 0x03, 0x0a, 0x0d, 0x4b, 0x65, + 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x2e, 0x0a, 0x03, 0x50, + 0x75, 0x74, 0x12, 0x12, 0x2e, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x50, 0x75, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, + 0x50, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x03, 0x47, + 0x65, 0x74, 0x12, 0x12, 0x2e, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x47, 0x65, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, + 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x05, 0x57, + 0x61, 0x74, 0x63, 0x68, 0x12, 0x14, 0x2e, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x57, 0x61, + 0x74, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x73, 0x79, 0x73, + 0x74, 0x65, 0x6d, 0x2e, 0x57, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x30, 0x01, 0x12, 0x37, 0x0a, 0x06, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x15, 0x2e, + 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x44, 0x65, + 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x08, + 0x4c, 0x69, 0x73, 0x74, 0x4b, 0x65, 0x79, 0x73, 0x12, 0x17, 0x2e, 0x73, 0x79, 0x73, 0x74, 0x65, + 0x6d, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4b, 0x65, 0x79, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x18, 0x2e, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4b, + 0x65, 0x79, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3a, 0x0a, 0x07, 0x48, + 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x16, 0x2e, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, + 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, + 0x2e, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x04, 0x4c, 0x6f, 0x63, 0x6b, 0x12, + 0x13, 0x2e, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x4c, 0x6f, 0x63, 0x6b, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x2e, 0x4c, 0x6f, + 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x42, 0x31, 0x5a, 0x2f, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x61, 0x6e, 0x63, 0x68, + 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x6e, 0x69, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x6c, 0x75, 0x67, + 0x69, 0x6e, 0x73, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1231,9 +1231,9 @@ var file_github_com_rancher_opni_pkg_plugins_apis_system_system_proto_depIdxs = 14, // 3: system.WatchResponse.current:type_name -> system.KeyRevision 14, // 4: system.WatchResponse.previous:type_name -> system.KeyRevision 1, // 5: system.LockResponse.event:type_name -> system.LockResponse.Event - 2, // 6: system.System.UseManagementAPI:input_type -> system.BrokerID - 2, // 7: system.System.UseKeyValueStore:input_type -> system.BrokerID - 3, // 8: system.System.UseAPIExtensions:input_type -> system.DialAddress + 2, // 6: system.System.UseConfigAPI:input_type -> system.BrokerID + 2, // 7: system.System.UseManagementAPI:input_type -> system.BrokerID + 2, // 8: system.System.UseKeyValueStore:input_type -> system.BrokerID 20, // 9: system.System.UseCachingProvider:input_type -> google.protobuf.Empty 4, // 10: system.KeyValueStore.Put:input_type -> system.PutRequest 6, // 11: system.KeyValueStore.Get:input_type -> system.GetRequest @@ -1242,9 +1242,9 @@ var file_github_com_rancher_opni_pkg_plugins_apis_system_system_proto_depIdxs = 10, // 14: system.KeyValueStore.ListKeys:input_type -> system.ListKeysRequest 12, // 15: system.KeyValueStore.History:input_type -> system.HistoryRequest 17, // 16: system.KeyValueStore.Lock:input_type -> system.LockRequest - 20, // 17: system.System.UseManagementAPI:output_type -> google.protobuf.Empty - 20, // 18: system.System.UseKeyValueStore:output_type -> google.protobuf.Empty - 20, // 19: system.System.UseAPIExtensions:output_type -> google.protobuf.Empty + 20, // 17: system.System.UseConfigAPI:output_type -> google.protobuf.Empty + 20, // 18: system.System.UseManagementAPI:output_type -> google.protobuf.Empty + 20, // 19: system.System.UseKeyValueStore:output_type -> google.protobuf.Empty 20, // 20: system.System.UseCachingProvider:output_type -> google.protobuf.Empty 5, // 21: system.KeyValueStore.Put:output_type -> system.PutResponse 7, // 22: system.KeyValueStore.Get:output_type -> system.GetResponse diff --git a/pkg/plugins/apis/system/system.proto b/pkg/plugins/apis/system/system.proto index d4a741198f..d9aa753cb1 100644 --- a/pkg/plugins/apis/system/system.proto +++ b/pkg/plugins/apis/system/system.proto @@ -9,9 +9,9 @@ import "google/protobuf/timestamp.proto"; option go_package = "github.com/rancher/opni/pkg/plugins/apis/system"; service System { + rpc UseConfigAPI(BrokerID) returns (google.protobuf.Empty); rpc UseManagementAPI(BrokerID) returns (google.protobuf.Empty); rpc UseKeyValueStore(BrokerID) returns (google.protobuf.Empty); - rpc UseAPIExtensions(DialAddress) returns (google.protobuf.Empty); rpc UseCachingProvider(google.protobuf.Empty) returns (google.protobuf.Empty); } diff --git a/pkg/plugins/apis/system/system_grpc.pb.go b/pkg/plugins/apis/system/system_grpc.pb.go index 575adb9c84..bcbd56aaee 100644 --- a/pkg/plugins/apis/system/system_grpc.pb.go +++ b/pkg/plugins/apis/system/system_grpc.pb.go @@ -20,9 +20,9 @@ import ( const _ = grpc.SupportPackageIsVersion7 const ( + System_UseConfigAPI_FullMethodName = "/system.System/UseConfigAPI" System_UseManagementAPI_FullMethodName = "/system.System/UseManagementAPI" System_UseKeyValueStore_FullMethodName = "/system.System/UseKeyValueStore" - System_UseAPIExtensions_FullMethodName = "/system.System/UseAPIExtensions" System_UseCachingProvider_FullMethodName = "/system.System/UseCachingProvider" ) @@ -30,9 +30,9 @@ const ( // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type SystemClient interface { + UseConfigAPI(ctx context.Context, in *BrokerID, opts ...grpc.CallOption) (*emptypb.Empty, error) UseManagementAPI(ctx context.Context, in *BrokerID, opts ...grpc.CallOption) (*emptypb.Empty, error) UseKeyValueStore(ctx context.Context, in *BrokerID, opts ...grpc.CallOption) (*emptypb.Empty, error) - UseAPIExtensions(ctx context.Context, in *DialAddress, opts ...grpc.CallOption) (*emptypb.Empty, error) UseCachingProvider(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*emptypb.Empty, error) } @@ -44,27 +44,27 @@ func NewSystemClient(cc grpc.ClientConnInterface) SystemClient { return &systemClient{cc} } -func (c *systemClient) UseManagementAPI(ctx context.Context, in *BrokerID, opts ...grpc.CallOption) (*emptypb.Empty, error) { +func (c *systemClient) UseConfigAPI(ctx context.Context, in *BrokerID, opts ...grpc.CallOption) (*emptypb.Empty, error) { out := new(emptypb.Empty) - err := c.cc.Invoke(ctx, System_UseManagementAPI_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, System_UseConfigAPI_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } -func (c *systemClient) UseKeyValueStore(ctx context.Context, in *BrokerID, opts ...grpc.CallOption) (*emptypb.Empty, error) { +func (c *systemClient) UseManagementAPI(ctx context.Context, in *BrokerID, opts ...grpc.CallOption) (*emptypb.Empty, error) { out := new(emptypb.Empty) - err := c.cc.Invoke(ctx, System_UseKeyValueStore_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, System_UseManagementAPI_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } -func (c *systemClient) UseAPIExtensions(ctx context.Context, in *DialAddress, opts ...grpc.CallOption) (*emptypb.Empty, error) { +func (c *systemClient) UseKeyValueStore(ctx context.Context, in *BrokerID, opts ...grpc.CallOption) (*emptypb.Empty, error) { out := new(emptypb.Empty) - err := c.cc.Invoke(ctx, System_UseAPIExtensions_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, System_UseKeyValueStore_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -84,9 +84,9 @@ func (c *systemClient) UseCachingProvider(ctx context.Context, in *emptypb.Empty // All implementations should embed UnimplementedSystemServer // for forward compatibility type SystemServer interface { + UseConfigAPI(context.Context, *BrokerID) (*emptypb.Empty, error) UseManagementAPI(context.Context, *BrokerID) (*emptypb.Empty, error) UseKeyValueStore(context.Context, *BrokerID) (*emptypb.Empty, error) - UseAPIExtensions(context.Context, *DialAddress) (*emptypb.Empty, error) UseCachingProvider(context.Context, *emptypb.Empty) (*emptypb.Empty, error) } @@ -94,15 +94,15 @@ type SystemServer interface { type UnimplementedSystemServer struct { } +func (UnimplementedSystemServer) UseConfigAPI(context.Context, *BrokerID) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method UseConfigAPI not implemented") +} func (UnimplementedSystemServer) UseManagementAPI(context.Context, *BrokerID) (*emptypb.Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method UseManagementAPI not implemented") } func (UnimplementedSystemServer) UseKeyValueStore(context.Context, *BrokerID) (*emptypb.Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method UseKeyValueStore not implemented") } -func (UnimplementedSystemServer) UseAPIExtensions(context.Context, *DialAddress) (*emptypb.Empty, error) { - return nil, status.Errorf(codes.Unimplemented, "method UseAPIExtensions not implemented") -} func (UnimplementedSystemServer) UseCachingProvider(context.Context, *emptypb.Empty) (*emptypb.Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method UseCachingProvider not implemented") } @@ -118,56 +118,56 @@ func RegisterSystemServer(s grpc.ServiceRegistrar, srv SystemServer) { s.RegisterService(&System_ServiceDesc, srv) } -func _System_UseManagementAPI_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { +func _System_UseConfigAPI_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(BrokerID) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(SystemServer).UseManagementAPI(ctx, in) + return srv.(SystemServer).UseConfigAPI(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: System_UseManagementAPI_FullMethodName, + FullMethod: System_UseConfigAPI_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SystemServer).UseManagementAPI(ctx, req.(*BrokerID)) + return srv.(SystemServer).UseConfigAPI(ctx, req.(*BrokerID)) } return interceptor(ctx, in, info, handler) } -func _System_UseKeyValueStore_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { +func _System_UseManagementAPI_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(BrokerID) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(SystemServer).UseKeyValueStore(ctx, in) + return srv.(SystemServer).UseManagementAPI(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: System_UseKeyValueStore_FullMethodName, + FullMethod: System_UseManagementAPI_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SystemServer).UseKeyValueStore(ctx, req.(*BrokerID)) + return srv.(SystemServer).UseManagementAPI(ctx, req.(*BrokerID)) } return interceptor(ctx, in, info, handler) } -func _System_UseAPIExtensions_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(DialAddress) +func _System_UseKeyValueStore_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(BrokerID) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(SystemServer).UseAPIExtensions(ctx, in) + return srv.(SystemServer).UseKeyValueStore(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: System_UseAPIExtensions_FullMethodName, + FullMethod: System_UseKeyValueStore_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SystemServer).UseAPIExtensions(ctx, req.(*DialAddress)) + return srv.(SystemServer).UseKeyValueStore(ctx, req.(*BrokerID)) } return interceptor(ctx, in, info, handler) } @@ -197,6 +197,10 @@ var System_ServiceDesc = grpc.ServiceDesc{ ServiceName: "system.System", HandlerType: (*SystemServer)(nil), Methods: []grpc.MethodDesc{ + { + MethodName: "UseConfigAPI", + Handler: _System_UseConfigAPI_Handler, + }, { MethodName: "UseManagementAPI", Handler: _System_UseManagementAPI_Handler, @@ -205,10 +209,6 @@ var System_ServiceDesc = grpc.ServiceDesc{ MethodName: "UseKeyValueStore", Handler: _System_UseKeyValueStore_Handler, }, - { - MethodName: "UseAPIExtensions", - Handler: _System_UseAPIExtensions_Handler, - }, { MethodName: "UseCachingProvider", Handler: _System_UseCachingProvider_Handler, diff --git a/pkg/storage/etcd/etcd.go b/pkg/storage/etcd/etcd.go index 9adc1b3993..e585d6b00d 100644 --- a/pkg/storage/etcd/etcd.go +++ b/pkg/storage/etcd/etcd.go @@ -73,7 +73,11 @@ func NewEtcdStore(ctx context.Context, conf *configv1.EtcdSpec, opts ...EtcdStor var err error tlsConfig, err = conf.Certs.AsTlsConfig() if err != nil { - return nil, fmt.Errorf("failed to load client TLS config: %w", err) + if errors.Is(err, configv1.ErrInsecure) { + lg.Warn(err.Error()) + } else { + return nil, fmt.Errorf("failed to load client TLS config: %w", err) + } } } clientConfig := clientv3.Config{ @@ -135,7 +139,7 @@ func (e *EtcdStore) LockManager(prefix string) storage.LockManager { } func init() { - storage.RegisterStoreBuilder(v1beta1.StorageTypeEtcd, func(args ...any) (any, error) { + storage.RegisterStoreBuilder(configv1.StorageBackend_Etcd.String(), func(args ...any) (any, error) { ctx := args[0].(context.Context) var conf *configv1.EtcdSpec diff --git a/pkg/storage/jetstream/jetstream.go b/pkg/storage/jetstream/jetstream.go index 27ee7bac4f..e6318a9635 100644 --- a/pkg/storage/jetstream/jetstream.go +++ b/pkg/storage/jetstream/jetstream.go @@ -263,7 +263,7 @@ func jetstreamGrpcError(err error) error { } func init() { - storage.RegisterStoreBuilder(v1beta1.StorageTypeJetStream, func(args ...any) (any, error) { + storage.RegisterStoreBuilder(configv1.StorageBackend_JetStream.String(), func(args ...any) (any, error) { ctx := args[0].(context.Context) var conf *configv1.JetStreamSpec diff --git a/pkg/storage/stores.go b/pkg/storage/stores.go index d0f6e2d619..72c64ffa45 100644 --- a/pkg/storage/stores.go +++ b/pkg/storage/stores.go @@ -2,6 +2,7 @@ package storage import ( "context" + "strings" "time" corev1 "github.com/rancher/opni/pkg/apis/core/v1" @@ -297,9 +298,9 @@ var ( ) func RegisterStoreBuilder[T ~string](name T, builder func(...any) (any, error)) { - storeBuilderCache[string(name)] = builder + storeBuilderCache[strings.ToLower(string(name))] = builder } func GetStoreBuilder[T ~string](name T) func(...any) (any, error) { - return storeBuilderCache[string(name)] + return storeBuilderCache[strings.ToLower(string(name))] } diff --git a/pkg/test/mock/apiextensions/apiextensions.go b/pkg/test/mock/apiextensions/apiextensions.go index 0b5c132f9e..8a0e80ba39 100644 --- a/pkg/test/mock/apiextensions/apiextensions.go +++ b/pkg/test/mock/apiextensions/apiextensions.go @@ -9,6 +9,7 @@ import ( reflect "reflect" totem "github.com/kralicky/totem" + v1 "github.com/rancher/opni/pkg/config/v1" apiextensions "github.com/rancher/opni/pkg/plugins/apis/apiextensions" gomock "go.uber.org/mock/gomock" grpc "google.golang.org/grpc" @@ -504,7 +505,7 @@ func (m *MockHTTPAPIExtensionClient) EXPECT() *MockHTTPAPIExtensionClientMockRec } // Configure mocks base method. -func (m *MockHTTPAPIExtensionClient) Configure(ctx context.Context, in *apiextensions.CertConfig, opts ...grpc.CallOption) (*apiextensions.HTTPAPIExtensionConfig, error) { +func (m *MockHTTPAPIExtensionClient) Configure(ctx context.Context, in *v1.CertsSpec, opts ...grpc.CallOption) (*apiextensions.HTTPAPIExtensionConfig, error) { m.ctrl.T.Helper() varargs := []interface{}{ctx, in} for _, a := range opts { @@ -547,7 +548,7 @@ func (m *MockHTTPAPIExtensionServer) EXPECT() *MockHTTPAPIExtensionServerMockRec } // Configure mocks base method. -func (m *MockHTTPAPIExtensionServer) Configure(arg0 context.Context, arg1 *apiextensions.CertConfig) (*apiextensions.HTTPAPIExtensionConfig, error) { +func (m *MockHTTPAPIExtensionServer) Configure(arg0 context.Context, arg1 *v1.CertsSpec) (*apiextensions.HTTPAPIExtensionConfig, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Configure", arg0, arg1) ret0, _ := ret[0].(*apiextensions.HTTPAPIExtensionConfig) diff --git a/pkg/update/kubernetes/server/server.go b/pkg/update/kubernetes/server/server.go index c75e250deb..5b6d455f32 100644 --- a/pkg/update/kubernetes/server/server.go +++ b/pkg/update/kubernetes/server/server.go @@ -4,10 +4,15 @@ import ( "context" "fmt" "os" + "sync/atomic" + + "log/slog" "github.com/prometheus/client_golang/prometheus" controlv1 "github.com/rancher/opni/pkg/apis/control/v1" - "github.com/rancher/opni/pkg/config/v1beta1" + "github.com/rancher/opni/pkg/config/reactive" + configv1 "github.com/rancher/opni/pkg/config/v1" + "github.com/rancher/opni/pkg/logger" "github.com/rancher/opni/pkg/machinery" "github.com/rancher/opni/pkg/oci" "github.com/rancher/opni/pkg/update" @@ -17,11 +22,10 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" - "log/slog" ) type kubernetesSyncServer struct { - imageFetcher oci.Fetcher + imageFetcher atomic.Pointer[oci.Fetcher] lg *slog.Logger } @@ -44,7 +48,8 @@ func WithNamespace(namespace string) KubernetesOption { } func NewKubernetesSyncServer( - conf v1beta1.KubernetesAgentUpgradeSpec, + ctx context.Context, + conf reactive.Reactive[*configv1.KubernetesAgentUpgradeSpec], lg *slog.Logger, opts ...KubernetesOption, ) (update.UpdateTypeHandler, error) { @@ -53,15 +58,22 @@ func NewKubernetesSyncServer( } options.apply(opts...) - imageFetcher, err := machinery.ConfigureOCIFetcher(string(conf.ImageResolver), options.namespace) - if err != nil { - return nil, err + srv := &kubernetesSyncServer{ + imageFetcher: atomic.Pointer[oci.Fetcher]{}, + lg: lg, } - return &kubernetesSyncServer{ - imageFetcher: imageFetcher, - lg: lg, - }, nil + conf.WatchFunc(ctx, func(conf *configv1.KubernetesAgentUpgradeSpec) { + imageFetcher, err := machinery.ConfigureOCIFetcher(conf.GetImageResolver(), options.namespace) + if err != nil { + lg.With(logger.Err(err)).Error("failed to configure OCI image fetcher") + return + } + lg.With("resolver", conf.GetImageResolver().String()).Info("configured OCI image fetcher") + srv.imageFetcher.Store(&imageFetcher) + }) + + return srv, nil } func (k *kubernetesSyncServer) Strategy() string { @@ -97,6 +109,10 @@ func (k *kubernetesSyncServer) CalculateExpectedManifest(ctx context.Context, up if updateType != opniurn.Agent { return nil, status.Error(codes.Unimplemented, kubernetes.ErrUnhandledUpdateType(string(updateType)).Error()) } + imageFetcher := k.imageFetcher.Load() + if imageFetcher == nil { + return nil, status.Error(codes.Unavailable, "oci image fetcher not configured") + } expectedManifest := &controlv1.UpdateManifest{} strategyKey := controlv1.UpdateStrategyKeyForType(updateType) strategy := metadata.ValueFromIncomingContext(ctx, strategyKey) @@ -106,7 +122,7 @@ func (k *kubernetesSyncServer) CalculateExpectedManifest(ctx context.Context, up return nil, status.Errorf(codes.InvalidArgument, "expected 1 value for metadata key %s, got %d", strategyKey, len(strategy)) } for component, updateType := range kubernetes.ComponentImageMap { - image, err := k.imageFetcher.GetImage(ctx, updateType) + image, err := (*imageFetcher).GetImage(ctx, updateType) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } @@ -164,6 +180,11 @@ func (k *kubernetesSyncServer) imageForEntry( ctx context.Context, entry *controlv1.UpdateManifestEntry, ) (*oci.Image, error) { + imageFetcher := k.imageFetcher.Load() + if imageFetcher == nil { + return nil, status.Error(codes.Unavailable, "oci image fetcher not configured") + } + urn, err := opniurn.ParseString(entry.GetPackage()) if err != nil { return nil, err @@ -177,7 +198,7 @@ func (k *kubernetesSyncServer) imageForEntry( return nil, nil } - return k.imageFetcher.GetImage(ctx, imageType) + return (*imageFetcher).GetImage(ctx, imageType) } func patchForEntry( diff --git a/pkg/update/kubernetes/server/server_test.go b/pkg/update/kubernetes/server/server_test.go index 0960965f98..98eba864a9 100644 --- a/pkg/update/kubernetes/server/server_test.go +++ b/pkg/update/kubernetes/server/server_test.go @@ -6,7 +6,9 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" controlv1 "github.com/rancher/opni/pkg/apis/control/v1" - "github.com/rancher/opni/pkg/config/v1beta1" + "github.com/rancher/opni/pkg/config/reactive" + "github.com/rancher/opni/pkg/config/reactive/reactivetest" + configv1 "github.com/rancher/opni/pkg/config/v1" _ "github.com/rancher/opni/pkg/oci/noop" "github.com/rancher/opni/pkg/test/testlog" "github.com/rancher/opni/pkg/update" @@ -16,6 +18,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" + "google.golang.org/protobuf/reflect/protopath" ) const ( @@ -29,10 +32,23 @@ var _ = Describe("Kubernetes sync server", Label("unit"), func() { )) BeforeEach(func() { + ctrl, ctx, ca := reactivetest.InMemoryController[*configv1.GatewayConfigSpec]( + reactivetest.WithExistingActiveConfig(&configv1.GatewayConfigSpec{ + Upgrades: &configv1.UpgradesSpec{ + Agents: &configv1.AgentUpgradesSpec{ + Driver: configv1.AgentUpgradesSpec_Kubernetes.Enum(), + }, + }, + }), + ) + DeferCleanup(ca) + var err error - k8sServer, err = server.NewKubernetesSyncServer(v1beta1.KubernetesAgentUpgradeSpec{ - ImageResolver: "noop", - }, testlog.Log) + k8sServer, err = server.NewKubernetesSyncServer(ctx, + reactive.Message[*configv1.KubernetesAgentUpgradeSpec]( + ctrl.Reactive(protopath.Path(configv1.ProtoPath().Upgrades().Agents().Kubernetes()))), + testlog.Log, + ) Expect(err).NotTo(HaveOccurred()) }) diff --git a/pkg/update/patch/filesystem.go b/pkg/update/patch/filesystem.go index e08ce08c5f..4329d23d30 100644 --- a/pkg/update/patch/filesystem.go +++ b/pkg/update/patch/filesystem.go @@ -12,7 +12,7 @@ import ( "github.com/klauspost/compress/zstd" controlv1 "github.com/rancher/opni/pkg/apis/control/v1" - "github.com/rancher/opni/pkg/config/v1beta1" + configv1 "github.com/rancher/opni/pkg/config/v1" "github.com/rancher/opni/pkg/logger" "github.com/spf13/afero" "golang.org/x/crypto/blake2b" @@ -27,7 +27,7 @@ const ( type FilesystemCache struct { CacheMetricsTracker - config v1beta1.FilesystemCacheSpec + config *configv1.FilesystemCacheSpec fs afero.Afero logger *slog.Logger cacheGroup singleflight.Group @@ -36,16 +36,16 @@ type FilesystemCache struct { var _ Cache = (*FilesystemCache)(nil) -func NewFilesystemCache(fsys afero.Fs, conf v1beta1.FilesystemCacheSpec, patcher BinaryPatcher, lg *slog.Logger) (Cache, error) { - if err := fsys.MkdirAll(conf.Dir, 0777); err != nil { +func NewFilesystemCache(fsys afero.Fs, conf *configv1.FilesystemCacheSpec, patcher BinaryPatcher, lg *slog.Logger) (Cache, error) { + if err := fsys.MkdirAll(conf.GetDir(), 0777); err != nil { lg.Error(fmt.Sprintf("failed to create cache directory: %v", err)) return nil, err } - if err := fsys.MkdirAll(filepath.Join(conf.Dir, PluginsDir), 0777); err != nil { + if err := fsys.MkdirAll(filepath.Join(conf.GetDir(), PluginsDir), 0777); err != nil { lg.Error(fmt.Sprintf("faIled to create plugins directory: %v", err)) return nil, err } - if err := fsys.MkdirAll(filepath.Join(conf.Dir, PatchesDir), 0777); err != nil { + if err := fsys.MkdirAll(filepath.Join(conf.GetDir(), PatchesDir), 0777); err != nil { lg.Error(fmt.Sprintf("failed to create patches directory: %v", err)) return nil, err } @@ -270,7 +270,7 @@ func (p *FilesystemCache) ListDigests() ([]string, error) { } func (p *FilesystemCache) path(parts ...string) string { - return filepath.Join(append([]string{p.config.Dir}, parts...)...) + return filepath.Join(append([]string{p.config.GetDir()}, parts...)...) } func (p *FilesystemCache) recomputeDiskStats() { diff --git a/pkg/update/patch/filesystem_test.go b/pkg/update/patch/filesystem_test.go index fbb8e95492..2f638edc5c 100644 --- a/pkg/update/patch/filesystem_test.go +++ b/pkg/update/patch/filesystem_test.go @@ -8,17 +8,18 @@ import ( . "github.com/onsi/gomega" "github.com/rancher/opni/pkg/test/memfs" "github.com/rancher/opni/pkg/test/testlog" + "github.com/samber/lo" "github.com/spf13/afero" - "github.com/rancher/opni/pkg/config/v1beta1" + configv1 "github.com/rancher/opni/pkg/config/v1" "github.com/rancher/opni/pkg/update/patch" ) func init() { BuildCacheTestSuite("Filesystem Cache", func() TestCache { fsys := afero.NewMemMapFs() - cache, err := patch.NewFilesystemCache(fsys, v1beta1.FilesystemCacheSpec{ - Dir: "/tmp", + cache, err := patch.NewFilesystemCache(fsys, &configv1.FilesystemCacheSpec{ + Dir: lo.ToPtr("/tmp"), }, patch.BsdiffPatcher{}, testlog.Log) Expect(err).NotTo(HaveOccurred()) return newTestCache(cache, CacheTestSuiteOptions{ @@ -51,29 +52,29 @@ var _ = Describe("Filesystem Cache", Label("unit"), func() { Expect(fs.Chmod(filepath.Join(tmpDir, "x"), 0)).To(Succeed()) - _, err := patch.NewFilesystemCache(fs, v1beta1.FilesystemCacheSpec{ - Dir: filepath.Join(tmpDir, "x"), + _, err := patch.NewFilesystemCache(fs, &configv1.FilesystemCacheSpec{ + Dir: lo.ToPtr(filepath.Join(tmpDir, "x")), }, patch.BsdiffPatcher{}, testlog.Log) Expect(err).To(HaveOccurred()) Expect(fs.Chmod(filepath.Join(tmpDir, "x"), 0o777)).To(Succeed()) - _, err = patch.NewFilesystemCache(fs, v1beta1.FilesystemCacheSpec{ - Dir: filepath.Join(tmpDir, "x"), + _, err = patch.NewFilesystemCache(fs, &configv1.FilesystemCacheSpec{ + Dir: lo.ToPtr(filepath.Join(tmpDir, "x")), }, patch.BsdiffPatcher{}, testlog.Log) Expect(err).To(HaveOccurred()) Expect(fs.Remove(filepath.Join(tmpDir, "x", patch.PluginsDir))).To(Succeed()) - _, err = patch.NewFilesystemCache(fs, v1beta1.FilesystemCacheSpec{ - Dir: filepath.Join(tmpDir, "x"), + _, err = patch.NewFilesystemCache(fs, &configv1.FilesystemCacheSpec{ + Dir: lo.ToPtr(filepath.Join(tmpDir, "x")), }, patch.BsdiffPatcher{}, testlog.Log) Expect(err).To(HaveOccurred()) Expect(fs.Remove(filepath.Join(tmpDir, "x", patch.PatchesDir))).To(Succeed()) - _, err = patch.NewFilesystemCache(fs, v1beta1.FilesystemCacheSpec{ - Dir: filepath.Join(tmpDir, "x"), + _, err = patch.NewFilesystemCache(fs, &configv1.FilesystemCacheSpec{ + Dir: lo.ToPtr(filepath.Join(tmpDir, "x")), }, patch.BsdiffPatcher{}, testlog.Log) Expect(err).NotTo(HaveOccurred()) }) diff --git a/pkg/update/patch/patcher.go b/pkg/update/patch/patcher.go index 773700b4d4..9bb5ee5f7e 100644 --- a/pkg/update/patch/patcher.go +++ b/pkg/update/patch/patcher.go @@ -2,11 +2,18 @@ package patch import ( "bytes" + "context" + "fmt" "io" + "log/slog" + "sync" "github.com/gabstv/go-bsdiff/pkg/bsdiff" "github.com/gabstv/go-bsdiff/pkg/bspatch" "github.com/klauspost/compress/zstd" + "github.com/rancher/opni/pkg/config/reactive" + configv1 "github.com/rancher/opni/pkg/config/v1" + "google.golang.org/protobuf/reflect/protoreflect" ) const ( @@ -111,3 +118,60 @@ func (ZstdPatcher) CheckFormat(reader io.ReaderAt) bool { } return false } + +func NewReactivePatchEngine(ctx context.Context, lg *slog.Logger, config reactive.Value) BinaryPatcher { + r := &reactivePatcher{} + var waitOnce sync.Once + wait := make(chan struct{}) + config.WatchFunc(ctx, func(v protoreflect.Value) { + r.mu.Lock() + defer r.mu.Unlock() + switch v.Enum() { + case configv1.PatchEngine_Bsdiff.Number(): + r.patcher = BsdiffPatcher{} + lg.With("engine", "bsdiff").Info("patch engine configured") + case configv1.PatchEngine_Zstd.Number(): + r.patcher = ZstdPatcher{} + lg.With("engine", "zstd").Info("patch engine configured") + default: + lg.With("engine", v.String()).Warn("unknown patch engine") + } + waitOnce.Do(func() { + close(wait) + }) + }) + <-wait + return r +} + +type reactivePatcher struct { + mu sync.Mutex + patcher BinaryPatcher +} + +func (r *reactivePatcher) GeneratePatch(old io.Reader, new io.Reader, patchOut io.Writer) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.patcher == nil { + return fmt.Errorf("no patch engine configured") + } + return r.patcher.GeneratePatch(old, new, patchOut) +} + +func (r *reactivePatcher) ApplyPatch(old io.Reader, patch io.Reader, newOut io.Writer) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.patcher == nil { + return fmt.Errorf("no patch engine configured") + } + return r.patcher.ApplyPatch(old, patch, newOut) +} + +func (r *reactivePatcher) CheckFormat(reader io.ReaderAt) bool { + r.mu.Lock() + defer r.mu.Unlock() + if r.patcher == nil { + return false + } + return r.patcher.CheckFormat(reader) +} diff --git a/pkg/update/patch/server/server.go b/pkg/update/patch/server/server.go index 80aea72b3f..f04ceac543 100644 --- a/pkg/update/patch/server/server.go +++ b/pkg/update/patch/server/server.go @@ -12,7 +12,7 @@ import ( "github.com/prometheus/client_golang/prometheus" controlv1 "github.com/rancher/opni/pkg/apis/control/v1" corev1 "github.com/rancher/opni/pkg/apis/core/v1" - "github.com/rancher/opni/pkg/config/v1beta1" + configv1 "github.com/rancher/opni/pkg/config/v1" "github.com/rancher/opni/pkg/logger" "github.com/rancher/opni/pkg/plugins" "github.com/rancher/opni/pkg/storage" @@ -29,7 +29,7 @@ type FilesystemPluginSyncServer struct { controlv1.UnsafeUpdateSyncServer SyncServerOptions logger *slog.Logger - config v1beta1.PluginsSpec + config *configv1.FilesystemCacheSpec loadMetadataOnce sync.Once manifest *controlv1.UpdateManifest patchCache patch.Cache @@ -61,7 +61,9 @@ func WithFs(fsys afero.Fs) SyncServerOption { } func NewFilesystemPluginSyncServer( - cfg v1beta1.PluginsSpec, + ctx context.Context, + cacheConfig *configv1.CacheSpec, + patchEngine patch.BinaryPatcher, lg *slog.Logger, opts ...SyncServerOption, ) (*FilesystemPluginSyncServer, error) { @@ -70,31 +72,21 @@ func NewFilesystemPluginSyncServer( } options.apply(opts...) - var patchEngine patch.BinaryPatcher - switch cfg.Binary.Cache.PatchEngine { - case v1beta1.PatchEngineBsdiff: - patchEngine = patch.BsdiffPatcher{} - case v1beta1.PatchEngineZstd: - patchEngine = patch.ZstdPatcher{} - default: - return nil, fmt.Errorf("unknown patch engine: %s", cfg.Binary.Cache.PatchEngine) - } - var cache patch.Cache - switch cfg.Binary.Cache.Backend { - case v1beta1.CacheBackendFilesystem: + switch cacheConfig.GetBackend() { + case configv1.CacheBackend_Filesystem: var err error - cache, err = patch.NewFilesystemCache(options.fsys, cfg.Binary.Cache.Filesystem, patchEngine, lg.WithGroup("cache")) + cache, err = patch.NewFilesystemCache(options.fsys, cacheConfig.GetFilesystem(), patchEngine, lg.WithGroup("cache")) if err != nil { return nil, err } default: - return nil, fmt.Errorf("unknown cache backend: %s", cfg.Binary.Cache.Backend) + return nil, fmt.Errorf("unknown cache backend: %s", cacheConfig.GetBackend().String()) } return &FilesystemPluginSyncServer{ SyncServerOptions: options, - config: cfg, + config: cacheConfig.GetFilesystem(), logger: lg, patchCache: cache, }, nil @@ -148,7 +140,7 @@ func (f *FilesystemPluginSyncServer) loadUpdateManifest() { panic("bug: tried to call loadUpdateManifest twice") } md, err := patch.GetFilesystemPlugins(plugins.DiscoveryConfig{ - Dir: f.config.Dir, + Dir: f.config.GetDir(), Fs: f.fsys, Logger: f.logger, Filters: f.filters, diff --git a/pkg/update/patch/server/server_suite_test.go b/pkg/update/patch/server/server_suite_test.go index bf455820fa..c706e443c7 100644 --- a/pkg/update/patch/server/server_suite_test.go +++ b/pkg/update/patch/server/server_suite_test.go @@ -11,7 +11,6 @@ import ( . "github.com/onsi/gomega" "github.com/onsi/gomega/gexec" controlv1 "github.com/rancher/opni/pkg/apis/control/v1" - "github.com/rancher/opni/pkg/config/v1beta1" _ "github.com/rancher/opni/pkg/test/setup" "github.com/rancher/opni/pkg/test/testutil" "github.com/rancher/opni/pkg/update/patch" @@ -32,6 +31,9 @@ var ( test2Path = "github.com/rancher/opni/pkg/test/testdata/patch/test2" test2Package = "urn:opni:plugin:binary:github.com/rancher/opni/pkg/test/testdata/patch/test2" + bsdiffPatcher = patch.BsdiffPatcher{} + zstdPatcher = patch.ZstdPatcher{} + testBinaries = map[string]map[string][]byte{ "test1": {}, "test2": {}, @@ -42,14 +44,14 @@ var ( "test2": {}, } - test1v1tov2Patch = map[v1beta1.PatchEngine]*bytes.Buffer{ - "bsdiff": new(bytes.Buffer), - "zstd": new(bytes.Buffer), + test1v1tov2Patch = map[patch.BinaryPatcher]*bytes.Buffer{ + bsdiffPatcher: new(bytes.Buffer), + zstdPatcher: new(bytes.Buffer), } - test2v1tov2Patch = map[v1beta1.PatchEngine]*bytes.Buffer{ - "bsdiff": new(bytes.Buffer), - "zstd": new(bytes.Buffer), + test2v1tov2Patch = map[patch.BinaryPatcher]*bytes.Buffer{ + bsdiffPatcher: new(bytes.Buffer), + zstdPatcher: new(bytes.Buffer), } ) @@ -121,19 +123,18 @@ var _ = BeforeSuite(func() { }) Expect(eg.Wait()).To(Succeed()) - patchers := map[v1beta1.PatchEngine]patch.BinaryPatcher{ - v1beta1.PatchEngineBsdiff: patch.BsdiffPatcher{}, - v1beta1.PatchEngineZstd: patch.ZstdPatcher{}, - } eg = errgroup.Group{} - for name, patcher := range patchers { - name, patcher := name, patcher + for _, patcher := range []patch.BinaryPatcher{ + bsdiffPatcher, + zstdPatcher, + } { + patcher := patcher eg.Go(func() error { return patcher.GeneratePatch( bytes.NewReader(testBinaries["test1"]["v1"]), bytes.NewReader(testBinaries["test1"]["v2"]), - test1v1tov2Patch[name], + test1v1tov2Patch[patcher], ) }) @@ -141,7 +142,7 @@ var _ = BeforeSuite(func() { return patcher.GeneratePatch( bytes.NewReader(testBinaries["test2"]["v1"]), bytes.NewReader(testBinaries["test2"]["v2"]), - test2v1tov2Patch[name], + test2v1tov2Patch[patcher], ) }) } diff --git a/pkg/update/patch/server/server_test.go b/pkg/update/patch/server/server_test.go index 46b9b7de0b..5639969ad0 100644 --- a/pkg/update/patch/server/server_test.go +++ b/pkg/update/patch/server/server_test.go @@ -16,7 +16,7 @@ import ( controlv1 "github.com/rancher/opni/pkg/apis/control/v1" corev1 "github.com/rancher/opni/pkg/apis/core/v1" "github.com/rancher/opni/pkg/auth/cluster" - "github.com/rancher/opni/pkg/config/v1beta1" + configv1 "github.com/rancher/opni/pkg/config/v1" "github.com/rancher/opni/pkg/test/memfs" mock_storage "github.com/rancher/opni/pkg/test/mock/storage" "github.com/rancher/opni/pkg/test/testgrpc" @@ -28,6 +28,7 @@ import ( "github.com/rancher/opni/pkg/update/patch/server" "github.com/rancher/opni/pkg/urn" "github.com/rancher/opni/pkg/util/streams" + "github.com/samber/lo" "github.com/spf13/afero" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -37,41 +38,29 @@ import ( "google.golang.org/grpc/test/bufconn" ) -var bsdiffSuiteSpec = v1beta1.CacheSpec{ - PatchEngine: v1beta1.PatchEngineBsdiff, - Backend: v1beta1.CacheBackendFilesystem, -} - -var zstdSuiteSpec = v1beta1.CacheSpec{ - PatchEngine: v1beta1.PatchEngineZstd, - Backend: v1beta1.CacheBackendFilesystem, -} - -var _ = Describe("Filesystem Sync Server (zstd)", Ordered, Label("unit", "slow"), FilesystemSyncServerTestSuite(zstdSuiteSpec)) -var _ = Describe("Filesystem Sync Server (bsdiff)", Ordered, Label("unit", "slow"), FilesystemSyncServerTestSuite(bsdiffSuiteSpec)) +var _ = Describe("Filesystem Sync Server (zstd)", Ordered, Label("unit", "slow"), FilesystemSyncServerTestSuite(zstdPatcher)) +var _ = Describe("Filesystem Sync Server (bsdiff)", Ordered, Label("unit", "slow"), FilesystemSyncServerTestSuite(bsdiffPatcher)) -func FilesystemSyncServerTestSuite(spec v1beta1.CacheSpec) func() { +func FilesystemSyncServerTestSuite(patcher patch.BinaryPatcher) func() { return func() { var srv *server.FilesystemPluginSyncServer var updateSrv *update.UpdateServer fsys := afero.Afero{Fs: memfs.NewModeAwareMemFs()} tmpDir := "/tmp/test" fsys.MkdirAll(tmpDir, 0755) - spec.Filesystem = v1beta1.FilesystemCacheSpec{ - Dir: filepath.Join(tmpDir, "cache"), - } var agentManifest *controlv1.UpdateManifest var srvManifestV1 *controlv1.UpdateManifest var srvManifestV2 *controlv1.UpdateManifest + cacheSpec := &configv1.CacheSpec{ + Backend: configv1.CacheBackend_Filesystem.Enum(), + Filesystem: &configv1.FilesystemCacheSpec{ + Dir: lo.ToPtr(filepath.Join(tmpDir, "cache")), + }, + } newServer := func() (*server.FilesystemPluginSyncServer, error) { - return server.NewFilesystemPluginSyncServer(v1beta1.PluginsSpec{ - Dir: filepath.Join(tmpDir, patch.PluginsDir), - Binary: v1beta1.BinaryPluginsSpec{ - Cache: spec, - }, - }, testlog.Log, server.WithFs(fsys)) + return server.NewFilesystemPluginSyncServer(context.Background(), cacheSpec, patcher, testlog.Log, server.WithFs(fsys)) } newUpdateServer := func(s *server.FilesystemPluginSyncServer) *update.UpdateServer { srv := update.NewUpdateServer(testlog.Log) @@ -179,14 +168,14 @@ func FilesystemSyncServerTestSuite(spec v1beta1.CacheSpec) func() { Expect(patches.Items[0].OldDigest).To(Equal(srvManifestV1.Items[0].Digest)) Expect(patches.Items[0].NewDigest).To(Equal(srvManifestV2.Items[0].Digest)) Expect(patches.Items[0].Path).To(Equal(srvManifestV1.Items[0].Path)) - Expect(patches.Items[0].Data).To(Equal(test1v1tov2Patch[spec.PatchEngine].Bytes())) + Expect(patches.Items[0].Data).To(Equal(test1v1tov2Patch[patcher].Bytes())) Expect(patches.Items[1].Package).To(Equal(test2Package)) Expect(patches.Items[1].Op).To(Equal(controlv1.PatchOp_Update)) Expect(patches.Items[1].OldDigest).To(Equal(srvManifestV1.Items[1].Digest)) Expect(patches.Items[1].NewDigest).To(Equal(srvManifestV2.Items[1].Digest)) Expect(patches.Items[1].Path).To(Equal(srvManifestV1.Items[1].Path)) - Expect(patches.Items[1].Data).To(Equal(test2v1tov2Patch[spec.PatchEngine].Bytes())) + Expect(patches.Items[1].Data).To(Equal(test2v1tov2Patch[patcher].Bytes())) initialPatchResponse = patches }) @@ -204,8 +193,8 @@ func FilesystemSyncServerTestSuite(spec v1beta1.CacheSpec) func() { } Expect(patches).To(ConsistOf( - test1v1tov2Patch[spec.PatchEngine].Bytes(), - test2v1tov2Patch[spec.PatchEngine].Bytes(), + test1v1tov2Patch[patcher].Bytes(), + test2v1tov2Patch[patcher].Bytes(), )) initialCacheItems = items @@ -473,47 +462,22 @@ func FilesystemSyncServerTestSuite(spec v1beta1.CacheSpec) func() { }) Context("error handling", func() { When("creating a new server", func() { - When("an unknown patch engine is specified", func() { - It("should return an error", func() { - _, err := server.NewFilesystemPluginSyncServer(v1beta1.PluginsSpec{ - Dir: tmpDir, - Binary: v1beta1.BinaryPluginsSpec{ - Cache: v1beta1.CacheSpec{ - PatchEngine: "unknown", - }, - }, - }, testlog.Log) - Expect(err).To(MatchError("unknown patch engine: unknown")) - }) - }) When("an unknown cache backend is specified", func() { It("should return an error", func() { - _, err := server.NewFilesystemPluginSyncServer(v1beta1.PluginsSpec{ - Dir: tmpDir, - Binary: v1beta1.BinaryPluginsSpec{ - Cache: v1beta1.CacheSpec{ - PatchEngine: spec.PatchEngine, - Backend: "unknown", - }, - }, - }, testlog.Log) + _, err := server.NewFilesystemPluginSyncServer(context.Background(), &configv1.CacheSpec{ + Backend: configv1.CacheBackend(2).Enum(), + }, patcher, testlog.Log) Expect(err).To(MatchError("unknown cache backend: unknown")) }) }) When("the filesystem cache cannot be created", func() { It("should return an error", func() { - _, err := server.NewFilesystemPluginSyncServer(v1beta1.PluginsSpec{ - Dir: tmpDir, - Binary: v1beta1.BinaryPluginsSpec{ - Cache: v1beta1.CacheSpec{ - PatchEngine: spec.PatchEngine, - Backend: v1beta1.CacheBackendFilesystem, - Filesystem: v1beta1.FilesystemCacheSpec{ - Dir: "/dev/null", - }, - }, + _, err := server.NewFilesystemPluginSyncServer(context.Background(), &configv1.CacheSpec{ + Backend: configv1.CacheBackend_Filesystem.Enum(), + Filesystem: &configv1.FilesystemCacheSpec{ + Dir: lo.ToPtr("/dev/null"), }, - }, testlog.Log) + }, patcher, testlog.Log) Expect(err).To(HaveOccurred()) }) }) diff --git a/pkg/update/update.go b/pkg/update/update.go index 90e84e3bfd..bf4e58c20f 100644 --- a/pkg/update/update.go +++ b/pkg/update/update.go @@ -2,6 +2,7 @@ package update import ( "context" + "strings" "sync" "github.com/prometheus/client_golang/prometheus" @@ -41,25 +42,25 @@ type SyncHandlerBuilder func(...any) (SyncHandler, error) func RegisterAgentSyncHandlerBuilder(strategy string, builder SyncHandlerBuilder) { agentSyncMu.Lock() defer agentSyncMu.Unlock() - agentSyncHandlerBuilders[strategy] = builder + agentSyncHandlerBuilders[strings.ToLower(strategy)] = builder } func RegisterPluginSyncHandlerBuilder(strategy string, builder SyncHandlerBuilder) { pluginSyncMu.Lock() defer pluginSyncMu.Unlock() - pluginSyncHandlerBuilders[strategy] = builder + pluginSyncHandlerBuilders[strings.ToLower(strategy)] = builder } func GetAgentSyncHandlerBuilder[T ~string](strategy T) SyncHandlerBuilder { agentSyncMu.Lock() defer agentSyncMu.Unlock() - return agentSyncHandlerBuilders[string(strategy)] + return agentSyncHandlerBuilders[strings.ToLower(string(strategy))] } func GetPluginSyncHandlerBuilder[T ~string](strategy T) SyncHandlerBuilder { pluginSyncMu.Lock() defer pluginSyncMu.Unlock() - return pluginSyncHandlerBuilders[string(strategy)] + return pluginSyncHandlerBuilders[strings.ToLower(string(strategy))] } type entry interface { diff --git a/pkg/util/listener.go b/pkg/util/listener.go index 4471842130..0ac30cfadc 100644 --- a/pkg/util/listener.go +++ b/pkg/util/listener.go @@ -8,11 +8,15 @@ import ( "net/url" "os" "path/filepath" + "strings" ) var ErrUnsupportedProtocolScheme = errors.New("unsupported protocol scheme") func NewProtocolListener(addr string) (net.Listener, error) { + if !strings.Contains(addr, "://") { + addr = "tcp://" + addr + } u, err := url.Parse(addr) if err != nil { return nil, err @@ -40,6 +44,9 @@ func NewProtocolListener(addr string) (net.Listener, error) { } func DialProtocol(ctx context.Context, addr string) (net.Conn, error) { + if !strings.Contains(addr, "://") { + addr = "tcp://" + addr + } u, err := url.Parse(addr) if err != nil { return nil, err diff --git a/pkg/util/notifier/notifier_test.go b/pkg/util/notifier/notifier_test.go index 110aaa393c..6368aa1543 100644 --- a/pkg/util/notifier/notifier_test.go +++ b/pkg/util/notifier/notifier_test.go @@ -129,7 +129,7 @@ var _ = Describe("Update Notifier", Label("unit"), func() { Eventually(channels[i]).Should(Receive(Equal(testGroups1))) } - groups.Store(testGroups2) // cancel the channels + groups.Store(&testGroups2) // cancel the channels for i := 0; i < count; i++ { contexts[i].ca() } @@ -160,7 +160,7 @@ var _ = Describe("Update Notifier", Label("unit"), func() { Eventually(channels[i]).Should(Receive(Equal(testGroups1))) } - groups.Store(testGroups2) + groups.Store(&testGroups2) go un.Refresh(context.Background()) @@ -180,7 +180,7 @@ var _ = Describe("Update Notifier", Label("unit"), func() { testGroups3_rulefmt = append(testGroups3_rulefmt, (rulefmt.RuleGroup)(group3)) } - groups.Store(testGroups3) + groups.Store(&testGroups3) go un.Refresh(context.Background()) diff --git a/pkg/util/servers.go b/pkg/util/servers.go index 4ed7f2810a..703ad3f814 100644 --- a/pkg/util/servers.go +++ b/pkg/util/servers.go @@ -35,7 +35,9 @@ func ServeHandler(ctx context.Context, handler http.Handler, listener net.Listen // context should be canceled. // 3. If a task exits successfully, the context should not be canceled and // other tasks should continue to run. -func WaitAll(ctx context.Context, ca context.CancelCauseFunc, channels ...<-chan error) error { +// +// Once WaitAll returns, the error can be retrieved using context.Cause(ctx). +func WaitAll(ctx context.Context, ca context.CancelCauseFunc, channels ...<-chan error) { cases := []reflect.SelectCase{ { Dir: reflect.SelectRecv, @@ -54,7 +56,7 @@ func WaitAll(ctx context.Context, ca context.CancelCauseFunc, channels ...<-chan for _, c := range channels { <-c } - return ctx.Err() + return } channelIdx := i - 1 var err error @@ -63,7 +65,8 @@ func WaitAll(ctx context.Context, ca context.CancelCauseFunc, channels ...<-chan } if err == nil { // run again, but skip the channel which exited successfully - return WaitAll(ctx, ca, append(channels[:channelIdx], channels[channelIdx+1:]...)...) + WaitAll(ctx, ca, append(channels[:channelIdx], channels[channelIdx+1:]...)...) + return } ca(err) for i, c := range channels { @@ -72,5 +75,4 @@ func WaitAll(ctx context.Context, ca context.CancelCauseFunc, channels ...<-chan } <-c } - return err } diff --git a/test/integration/agent_mem_test.go b/test/integration/agent_mem_test.go index 320d3c4f41..8257af8486 100644 --- a/test/integration/agent_mem_test.go +++ b/test/integration/agent_mem_test.go @@ -204,7 +204,15 @@ var _ = Describe("Agent Memory Tests", Ordered, Serial, Label("integration", "sl IdentityProvider: "env", Storage: v1beta1.StorageSpec{ Type: v1beta1.StorageTypeEtcd, - Etcd: environment.EtcdConfig(), + Etcd: &v1beta1.EtcdStorageSpec{ + Endpoints: environment.EtcdConfig().Endpoints, + Certs: &v1beta1.MTLSSpec{ + ServerCA: environment.EtcdConfig().Certs.GetServerCA(), + ClientCA: environment.EtcdConfig().Certs.GetClientCA(), + ClientCert: environment.EtcdConfig().Certs.GetClientCert(), + ClientKey: environment.EtcdConfig().Certs.GetClientKey(), + }, + }, }, Bootstrap: &v1beta1.BootstrapSpec{ Token: t.EncodeHex(), diff --git a/test/plugins/metrics/config_storage_test.go b/test/plugins/metrics/config_storage_test.go index 770edc4816..0027bfd07a 100644 --- a/test/plugins/metrics/config_storage_test.go +++ b/test/plugins/metrics/config_storage_test.go @@ -9,6 +9,7 @@ import ( . "github.com/onsi/gomega" "github.com/rancher/opni/apis" opnicorev1beta1 "github.com/rancher/opni/apis/core/v1beta1" + configv1 "github.com/rancher/opni/pkg/config/v1" "github.com/rancher/opni/pkg/config/v1beta1" "github.com/rancher/opni/pkg/machinery" "github.com/rancher/opni/pkg/plugins/apis/system" @@ -45,9 +46,9 @@ var _ = Describe("Config Storage Tests", Label("integration"), func() { }) Expect(err).NotTo(HaveOccurred()) - backend, err := machinery.ConfigureStorageBackend(environment.Context(), &v1beta1.StorageSpec{ - Type: v1beta1.StorageTypeEtcd, - Etcd: environment.EtcdConfig(), + backend, err := machinery.ConfigureStorageBackendV1(environment.Context(), &configv1.StorageSpec{ + Backend: configv1.StorageBackend_Etcd.Enum(), + Etcd: environment.EtcdConfig(), }) Expect(err).NotTo(HaveOccurred()) @@ -94,9 +95,9 @@ var _ = Describe("Config Storage Tests", Label("integration"), func() { Expect(environment.Start(test.WithEnableEtcd(true), test.WithEnableJetstream(false), test.WithStorageBackend(v1beta1.StorageTypeEtcd), test.WithEnableGateway(false))).To(Succeed()) DeferCleanup(environment.Stop) - backend, err := machinery.ConfigureStorageBackend(environment.Context(), &v1beta1.StorageSpec{ - Type: v1beta1.StorageTypeEtcd, - Etcd: environment.EtcdConfig(), + backend, err := machinery.ConfigureStorageBackendV1(environment.Context(), &configv1.StorageSpec{ + Backend: configv1.StorageBackend_Etcd.Enum(), + Etcd: environment.EtcdConfig(), }) Expect(err).NotTo(HaveOccurred())