extensions: authenticate allocation endpoint with requestheader client CA

cmd/extensions/main.go

@@ -17,6 +17,7 @@ package main
 
 import (
 	"context"
+	"crypto/x509"
 	"fmt"
 	"io"
 	"os"
@@ -47,6 +48,7 @@ import (
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
 	"gopkg.in/natefinch/lumberjack.v2"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 )
@@ -152,6 +154,16 @@ func main() {
 	wh := webhooks.NewWebHook(httpsServer.Mux)
 	api := apiserver.NewAPIServer(httpsServer.Mux)
 
+	// Load the requestheader client CA published by Kubernetes so that only the
+	// API server aggregation-layer proxy can reach allocation resource handlers.
+	// See: https://kubernetes.io/docs/tasks/extend-kubernetes/configure-aggregation-layer/
+	if ca, caErr := loadRequestHeaderCA(ctx, kubeClient); caErr != nil {
+		logger.WithError(caErr).Warn("Could not load requestheader client CA; allocation endpoint will reject unauthenticated requests once the CA is available")
+	} else {
+		api.SetRequestHeaderCA(ca)
+		logger.Info("Requestheader client CA loaded; allocation endpoint authentication enabled")
+	}
+
 	agonesInformerFactory := externalversions.NewSharedInformerFactory(agonesClient, defaultResync)
 	kubeInformerFactory := informers.NewSharedInformerFactory(kubeClient, defaultResync)
 
@@ -356,6 +368,26 @@ func parseEnvFlags() config {
 	}
 }
 
+// loadRequestHeaderCA fetches the requestheader client CA from the
+// kube-system/extension-apiserver-authentication ConfigMap, which Kubernetes
+// populates automatically for aggregated API servers.
+// See https://kubernetes.io/docs/tasks/extend-kubernetes/configure-aggregation-layer/
+func loadRequestHeaderCA(ctx context.Context, client kubernetes.Interface) (*x509.CertPool, error) {
+	cm, err := client.CoreV1().ConfigMaps("kube-system").Get(ctx, "extension-apiserver-authentication", metav1.GetOptions{})
+	if err != nil {
+		return nil, errors.Wrap(err, "getting extension-apiserver-authentication ConfigMap")
+	}
+	pemData, ok := cm.Data["requestheader-client-ca-file"]
+	if !ok || pemData == "" {
+		return nil, errors.New("requestheader-client-ca-file key missing or empty in extension-apiserver-authentication ConfigMap")
+	}
+	pool := x509.NewCertPool()
+	if !pool.AppendCertsFromPEM([]byte(pemData)) {
+		return nil, errors.New("no valid PEM certificates found in requestheader-client-ca-file")
+	}
+	return pool, nil
+}
+
 // config stores all required configuration to create a game server extensions.
 type config struct {
 	KeyFile    string

cmd/extensions/main_test.go

@@ -0,0 +1,114 @@
+// Copyright Contributors to Agones a Series of LF Projects, LLC.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kubefake "k8s.io/client-go/kubernetes/fake"
+)
+
+func selfSignedCAPEM(t *testing.T) string {
+	t.Helper()
+	key, err := rsa.GenerateKey(rand.Reader, 2048)
+	require.NoError(t, err)
+
+	tmpl := &x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		Subject:               pkix.Name{CommonName: "test-ca"},
+		NotBefore:             time.Now().Add(-time.Hour),
+		NotAfter:              time.Now().Add(time.Hour),
+		IsCA:                  true,
+		KeyUsage:              x509.KeyUsageCertSign,
+		BasicConstraintsValid: true,
+	}
+	certDER, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	require.NoError(t, err)
+
+	return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}))
+}
+
+func TestLoadRequestHeaderCA(t *testing.T) {
+	t.Parallel()
+
+	t.Run("valid CA in ConfigMap", func(t *testing.T) {
+		t.Parallel()
+		caPEM := selfSignedCAPEM(t)
+		client := kubefake.NewSimpleClientset(&corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "extension-apiserver-authentication",
+				Namespace: "kube-system",
+			},
+			Data: map[string]string{
+				"requestheader-client-ca-file": caPEM,
+			},
+		})
+		pool, err := loadRequestHeaderCA(context.Background(), client)
+		require.NoError(t, err)
+		assert.NotNil(t, pool)
+	})
+
+	t.Run("ConfigMap not found", func(t *testing.T) {
+		t.Parallel()
+		client := kubefake.NewSimpleClientset()
+		pool, err := loadRequestHeaderCA(context.Background(), client)
+		assert.Error(t, err)
+		assert.Nil(t, pool)
+	})
+
+	t.Run("requestheader-client-ca-file key missing", func(t *testing.T) {
+		t.Parallel()
+		client := kubefake.NewSimpleClientset(&corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "extension-apiserver-authentication",
+				Namespace: "kube-system",
+			},
+			Data: map[string]string{
+				"other-key": "value",
+			},
+		})
+		pool, err := loadRequestHeaderCA(context.Background(), client)
+		assert.Error(t, err)
+		assert.Nil(t, pool)
+	})
+
+	t.Run("invalid PEM data", func(t *testing.T) {
+		t.Parallel()
+		client := kubefake.NewSimpleClientset(&corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "extension-apiserver-authentication",
+				Namespace: "kube-system",
+			},
+			Data: map[string]string{
+				"requestheader-client-ca-file": "not-valid-pem",
+			},
+		})
+		pool, err := loadRequestHeaderCA(context.Background(), client)
+		assert.Error(t, err)
+		assert.Nil(t, pool)
+	})
+}

pkg/util/apiserver/apiserver.go

@@ -16,6 +16,7 @@
 package apiserver
 
 import (
+	"crypto/x509"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -76,6 +77,11 @@ type APIServer struct {
 	openapiv2          *spec.Swagger
 	openapiv3Discovery *handler3.OpenAPIV3Discovery
 	delegates          map[string]CRDHandler
+	// requestHeaderCA, when set, is the CA used to verify the requestheader proxy
+	// client certificate sent by the Kubernetes API server aggregation layer.
+	// Resource handlers require a valid client cert signed by this CA; discovery
+	// and OpenAPI handlers are left unauthenticated (they carry no sensitive data).
+	requestHeaderCA *x509.CertPool
 }
 
 // NewAPIServer returns a new API Server from the given Mux.
@@ -126,6 +132,43 @@ func NewAPIServer(mux *http.ServeMux) *APIServer {
 	return s
 }
 
+// SetRequestHeaderCA configures the CA pool used to authenticate the Kubernetes
+// aggregation-layer proxy client certificate.  Call this before any resource
+// handlers are invoked; requests that arrive without a certificate signed by
+// this CA will be rejected with HTTP 401.
+//
+// The verification logic mirrors k8s.io/apiserver/pkg/authentication/request/x509:
+// the leaf certificate from req.TLS.PeerCertificates is verified against the
+// supplied CA pool with ExtKeyUsageClientAuth.
+func (as *APIServer) SetRequestHeaderCA(ca *x509.CertPool) {
+	as.requestHeaderCA = ca
+}
+
+// authenticatedHandler wraps an ErrorHandlerFunc so that only requests carrying
+// a valid requestheader proxy certificate are forwarded.  If no CA has been
+// configured the handler is called directly (useful in unit tests).
+func (as *APIServer) authenticatedHandler(h https.ErrorHandlerFunc) https.ErrorHandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) error {
+		if as.requestHeaderCA == nil {
+			return h(w, r)
+		}
+		if r.TLS == nil || len(r.TLS.PeerCertificates) == 0 {
+			http.Error(w, "Unauthorized", http.StatusUnauthorized)
+			return nil
+		}
+		opts := x509.VerifyOptions{
+			Roots:     as.requestHeaderCA,
+			KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+		}
+		if _, err := r.TLS.PeerCertificates[0].Verify(opts); err != nil {
+			as.logger.WithError(err).Warn("rejecting request: requestheader client certificate verification failed")
+			http.Error(w, "Unauthorized", http.StatusUnauthorized)
+			return nil
+		}
+		return h(w, r)
+	}
+}
+
 // AddAPIResource stores the APIResource under the given groupVersion string, and returns it
 // in the appropriate place for the K8s discovery service
 // e.g. http://localhost:8001/apis/scheduling.k8s.io/v1
@@ -141,9 +184,10 @@ func (as *APIServer) AddAPIResource(groupVersion string, resource metav1.APIReso
 		as.logger.WithField("groupversion", groupVersion).WithField("pattern", pattern).Debug("Adding Discovery Handler")
 
 		// e.g.  /apis/agones.dev/v1/namespaces/default/gameservers
-		// CRD handler
+		// CRD handler — wrapped with requestheader client-cert authentication so that
+		// in-cluster workloads cannot bypass Kubernetes RBAC by calling this port directly.
 		pattern = fmt.Sprintf("/apis/%s/namespaces/", groupVersion)
-		as.mux.HandleFunc(pattern, https.ErrorHTTPHandler(as.logger, as.resourceHandler(groupVersion)))
+		as.mux.HandleFunc(pattern, https.ErrorHTTPHandler(as.logger, as.authenticatedHandler(as.resourceHandler(groupVersion))))
 		as.logger.WithField("groupversion", groupVersion).WithField("pattern", pattern).Debug("Adding Resource Handler")
 	}