diff --git a/cmd/extensions/main.go b/cmd/extensions/main.go index 4154b857b8..1edec16018 100644 --- a/cmd/extensions/main.go +++ b/cmd/extensions/main.go @@ -17,6 +17,7 @@ package main import ( "context" + "crypto/x509" "fmt" "io" "os" @@ -47,6 +48,7 @@ import ( "github.com/spf13/pflag" "github.com/spf13/viper" "gopkg.in/natefinch/lumberjack.v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" ) @@ -152,6 +154,16 @@ func main() { wh := webhooks.NewWebHook(httpsServer.Mux) api := apiserver.NewAPIServer(httpsServer.Mux) + // Load the requestheader client CA published by Kubernetes so that only the + // API server aggregation-layer proxy can reach allocation resource handlers. + // See: https://kubernetes.io/docs/tasks/extend-kubernetes/configure-aggregation-layer/ + if ca, caErr := loadRequestHeaderCA(ctx, kubeClient); caErr != nil { + logger.WithError(caErr).Warn("Could not load requestheader client CA; allocation endpoint will reject unauthenticated requests once the CA is available") + } else { + api.SetRequestHeaderCA(ca) + logger.Info("Requestheader client CA loaded; allocation endpoint authentication enabled") + } + agonesInformerFactory := externalversions.NewSharedInformerFactory(agonesClient, defaultResync) kubeInformerFactory := informers.NewSharedInformerFactory(kubeClient, defaultResync) @@ -356,6 +368,26 @@ func parseEnvFlags() config { } } +// loadRequestHeaderCA fetches the requestheader client CA from the +// kube-system/extension-apiserver-authentication ConfigMap, which Kubernetes +// populates automatically for aggregated API servers. +// See https://kubernetes.io/docs/tasks/extend-kubernetes/configure-aggregation-layer/ +func loadRequestHeaderCA(ctx context.Context, client kubernetes.Interface) (*x509.CertPool, error) { + cm, err := client.CoreV1().ConfigMaps("kube-system").Get(ctx, "extension-apiserver-authentication", metav1.GetOptions{}) + if err != nil { + return nil, errors.Wrap(err, "getting extension-apiserver-authentication ConfigMap") + } + pemData, ok := cm.Data["requestheader-client-ca-file"] + if !ok || pemData == "" { + return nil, errors.New("requestheader-client-ca-file key missing or empty in extension-apiserver-authentication ConfigMap") + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM([]byte(pemData)) { + return nil, errors.New("no valid PEM certificates found in requestheader-client-ca-file") + } + return pool, nil +} + // config stores all required configuration to create a game server extensions. type config struct { KeyFile string diff --git a/cmd/extensions/main_test.go b/cmd/extensions/main_test.go new file mode 100644 index 0000000000..26dad69934 --- /dev/null +++ b/cmd/extensions/main_test.go @@ -0,0 +1,114 @@ +// Copyright Contributors to Agones a Series of LF Projects, LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubefake "k8s.io/client-go/kubernetes/fake" +) + +func selfSignedCAPEM(t *testing.T) string { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + certDER, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + require.NoError(t, err) + + return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})) +} + +func TestLoadRequestHeaderCA(t *testing.T) { + t.Parallel() + + t.Run("valid CA in ConfigMap", func(t *testing.T) { + t.Parallel() + caPEM := selfSignedCAPEM(t) + client := kubefake.NewSimpleClientset(&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "extension-apiserver-authentication", + Namespace: "kube-system", + }, + Data: map[string]string{ + "requestheader-client-ca-file": caPEM, + }, + }) + pool, err := loadRequestHeaderCA(context.Background(), client) + require.NoError(t, err) + assert.NotNil(t, pool) + }) + + t.Run("ConfigMap not found", func(t *testing.T) { + t.Parallel() + client := kubefake.NewSimpleClientset() + pool, err := loadRequestHeaderCA(context.Background(), client) + assert.Error(t, err) + assert.Nil(t, pool) + }) + + t.Run("requestheader-client-ca-file key missing", func(t *testing.T) { + t.Parallel() + client := kubefake.NewSimpleClientset(&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "extension-apiserver-authentication", + Namespace: "kube-system", + }, + Data: map[string]string{ + "other-key": "value", + }, + }) + pool, err := loadRequestHeaderCA(context.Background(), client) + assert.Error(t, err) + assert.Nil(t, pool) + }) + + t.Run("invalid PEM data", func(t *testing.T) { + t.Parallel() + client := kubefake.NewSimpleClientset(&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "extension-apiserver-authentication", + Namespace: "kube-system", + }, + Data: map[string]string{ + "requestheader-client-ca-file": "not-valid-pem", + }, + }) + pool, err := loadRequestHeaderCA(context.Background(), client) + assert.Error(t, err) + assert.Nil(t, pool) + }) +} diff --git a/pkg/util/apiserver/apiserver.go b/pkg/util/apiserver/apiserver.go index 140f9765d8..ed495767c2 100644 --- a/pkg/util/apiserver/apiserver.go +++ b/pkg/util/apiserver/apiserver.go @@ -16,6 +16,7 @@ package apiserver import ( + "crypto/x509" "encoding/json" "fmt" "net/http" @@ -76,6 +77,11 @@ type APIServer struct { openapiv2 *spec.Swagger openapiv3Discovery *handler3.OpenAPIV3Discovery delegates map[string]CRDHandler + // requestHeaderCA, when set, is the CA used to verify the requestheader proxy + // client certificate sent by the Kubernetes API server aggregation layer. + // Resource handlers require a valid client cert signed by this CA; discovery + // and OpenAPI handlers are left unauthenticated (they carry no sensitive data). + requestHeaderCA *x509.CertPool } // NewAPIServer returns a new API Server from the given Mux. @@ -126,6 +132,43 @@ func NewAPIServer(mux *http.ServeMux) *APIServer { return s } +// SetRequestHeaderCA configures the CA pool used to authenticate the Kubernetes +// aggregation-layer proxy client certificate. Call this before any resource +// handlers are invoked; requests that arrive without a certificate signed by +// this CA will be rejected with HTTP 401. +// +// The verification logic mirrors k8s.io/apiserver/pkg/authentication/request/x509: +// the leaf certificate from req.TLS.PeerCertificates is verified against the +// supplied CA pool with ExtKeyUsageClientAuth. +func (as *APIServer) SetRequestHeaderCA(ca *x509.CertPool) { + as.requestHeaderCA = ca +} + +// authenticatedHandler wraps an ErrorHandlerFunc so that only requests carrying +// a valid requestheader proxy certificate are forwarded. If no CA has been +// configured the handler is called directly (useful in unit tests). +func (as *APIServer) authenticatedHandler(h https.ErrorHandlerFunc) https.ErrorHandlerFunc { + return func(w http.ResponseWriter, r *http.Request) error { + if as.requestHeaderCA == nil { + return h(w, r) + } + if r.TLS == nil || len(r.TLS.PeerCertificates) == 0 { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return nil + } + opts := x509.VerifyOptions{ + Roots: as.requestHeaderCA, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + } + if _, err := r.TLS.PeerCertificates[0].Verify(opts); err != nil { + as.logger.WithError(err).Warn("rejecting request: requestheader client certificate verification failed") + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return nil + } + return h(w, r) + } +} + // AddAPIResource stores the APIResource under the given groupVersion string, and returns it // in the appropriate place for the K8s discovery service // e.g. http://localhost:8001/apis/scheduling.k8s.io/v1 @@ -141,9 +184,10 @@ func (as *APIServer) AddAPIResource(groupVersion string, resource metav1.APIReso as.logger.WithField("groupversion", groupVersion).WithField("pattern", pattern).Debug("Adding Discovery Handler") // e.g. /apis/agones.dev/v1/namespaces/default/gameservers - // CRD handler + // CRD handler — wrapped with requestheader client-cert authentication so that + // in-cluster workloads cannot bypass Kubernetes RBAC by calling this port directly. pattern = fmt.Sprintf("/apis/%s/namespaces/", groupVersion) - as.mux.HandleFunc(pattern, https.ErrorHTTPHandler(as.logger, as.resourceHandler(groupVersion))) + as.mux.HandleFunc(pattern, https.ErrorHTTPHandler(as.logger, as.authenticatedHandler(as.resourceHandler(groupVersion)))) as.logger.WithField("groupversion", groupVersion).WithField("pattern", pattern).Debug("Adding Resource Handler") } diff --git a/pkg/util/apiserver/apiserver_test.go b/pkg/util/apiserver/apiserver_test.go index 328b5e78db..81ebc81160 100644 --- a/pkg/util/apiserver/apiserver_test.go +++ b/pkg/util/apiserver/apiserver_test.go @@ -15,11 +15,18 @@ package apiserver import ( + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" "encoding/json" "io" + "math/big" "net/http" "net/http/httptest" "testing" + "time" "github.com/go-openapi/spec" "github.com/stretchr/testify/assert" @@ -296,3 +303,154 @@ func TestSplitNameSpaceResource(t *testing.T) { }) } } + +func newTestCA(t *testing.T) (*x509.Certificate, *rsa.PrivateKey, *x509.CertPool) { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test-requestheader-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + BasicConstraintsValid: true, + } + certDER, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + require.NoError(t, err) + cert, err := x509.ParseCertificate(certDER) + require.NoError(t, err) + + pool := x509.NewCertPool() + pool.AddCert(cert) + return cert, key, pool +} + +func newClientCert(t *testing.T, caCert *x509.Certificate, caKey *rsa.PrivateKey) *x509.Certificate { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: "test-apiserver-proxy"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + } + certDER, err := x509.CreateCertificate(rand.Reader, tmpl, caCert, &key.PublicKey, caKey) + require.NoError(t, err) + cert, err := x509.ParseCertificate(certDER) + require.NoError(t, err) + return cert +} + +func TestAuthenticatedHandlerNoCA(t *testing.T) { + t.Parallel() + mux := http.NewServeMux() + api := NewAPIServer(mux) + + called := false + h := api.authenticatedHandler(func(w http.ResponseWriter, _ *http.Request) error { + called = true + w.WriteHeader(http.StatusOK) + return nil + }) + + req := httptest.NewRequest(http.MethodGet, "/test", nil) + rr := httptest.NewRecorder() + require.NoError(t, h(rr, req)) + assert.True(t, called) + assert.Equal(t, http.StatusOK, rr.Code) +} + +func TestAuthenticatedHandlerRejectPlainHTTP(t *testing.T) { + t.Parallel() + _, _, pool := newTestCA(t) + mux := http.NewServeMux() + api := NewAPIServer(mux) + api.SetRequestHeaderCA(pool) + + called := false + h := api.authenticatedHandler(func(_ http.ResponseWriter, _ *http.Request) error { + called = true + return nil + }) + + req := httptest.NewRequest(http.MethodGet, "/test", nil) + rr := httptest.NewRecorder() + require.NoError(t, h(rr, req)) + assert.False(t, called) + assert.Equal(t, http.StatusUnauthorized, rr.Code) +} + +func TestAuthenticatedHandlerRejectNoCert(t *testing.T) { + t.Parallel() + _, _, pool := newTestCA(t) + mux := http.NewServeMux() + api := NewAPIServer(mux) + api.SetRequestHeaderCA(pool) + + called := false + h := api.authenticatedHandler(func(_ http.ResponseWriter, _ *http.Request) error { + called = true + return nil + }) + + req := httptest.NewRequest(http.MethodGet, "/test", nil) + req.TLS = &tls.ConnectionState{} // no PeerCertificates + rr := httptest.NewRecorder() + require.NoError(t, h(rr, req)) + assert.False(t, called) + assert.Equal(t, http.StatusUnauthorized, rr.Code) +} + +func TestAuthenticatedHandlerAcceptValidCert(t *testing.T) { + t.Parallel() + caCert, caKey, pool := newTestCA(t) + clientCert := newClientCert(t, caCert, caKey) + + mux := http.NewServeMux() + api := NewAPIServer(mux) + api.SetRequestHeaderCA(pool) + + called := false + h := api.authenticatedHandler(func(w http.ResponseWriter, _ *http.Request) error { + called = true + w.WriteHeader(http.StatusOK) + return nil + }) + + req := httptest.NewRequest(http.MethodGet, "/test", nil) + req.TLS = &tls.ConnectionState{PeerCertificates: []*x509.Certificate{clientCert}} + rr := httptest.NewRecorder() + require.NoError(t, h(rr, req)) + assert.True(t, called) + assert.Equal(t, http.StatusOK, rr.Code) +} + +func TestAuthenticatedHandlerRejectUntrustedCert(t *testing.T) { + t.Parallel() + _, _, pool := newTestCA(t) + otherCA, otherKey, _ := newTestCA(t) + untrustedCert := newClientCert(t, otherCA, otherKey) + + mux := http.NewServeMux() + api := NewAPIServer(mux) + api.SetRequestHeaderCA(pool) + + called := false + h := api.authenticatedHandler(func(_ http.ResponseWriter, _ *http.Request) error { + called = true + return nil + }) + + req := httptest.NewRequest(http.MethodGet, "/test", nil) + req.TLS = &tls.ConnectionState{PeerCertificates: []*x509.Certificate{untrustedCert}} + rr := httptest.NewRecorder() + require.NoError(t, h(rr, req)) + assert.False(t, called) + assert.Equal(t, http.StatusUnauthorized, rr.Code) +} diff --git a/pkg/util/https/server.go b/pkg/util/https/server.go index caf5963328..3fd86c600c 100644 --- a/pkg/util/https/server.go +++ b/pkg/util/https/server.go @@ -79,6 +79,11 @@ func (s *Server) setupServer() { Handler: s.Mux, TLSConfig: &cryptotls.Config{ GetCertificate: s.getCertificate, + // RequestClientCert asks clients to send a certificate but does not + // require one. This makes the cert available in r.TLS.PeerCertificates + // so that the aggregation-layer auth middleware can verify it, without + // breaking webhook callers that do not present a cert. + ClientAuth: cryptotls.RequestClientCert, }, }