From 35dadde40efd6da596301a30eeae4d9f701e9576 Mon Sep 17 00:00:00 2001 From: Cloud-Architect-Emma Date: Mon, 1 Jun 2026 20:39:01 +0100 Subject: [PATCH] chore: promote PortPolicyNone feature gate from Beta to Stable - Move FeaturePortPolicyNone from Beta to Stable in features.go - Remove FeaturePortPolicyNone guards from gameserver.go and gke.go - Remove feature-gate-disabled test cases from unit tests - Remove t.Skip guard from e2e TestGameServerPortPolicyNone - Move PortPolicyNone to Stable section in defaultfeaturegates.yaml - Remove PortPolicyNone from BETA_FEATURE_GATES in Makefile - Remove PortPolicyNone=false from inverted e2e config in cloudbuild.yaml - Move row to Stable table in feature-stages.md - Add stable entry in versionMap.yaml for 1.59.0 Co-authored-by: Cloud-Architect-Emma Signed-off-by: Cloud-Architect-Emma --- build/Makefile | 3 +- cloudbuild.yaml | 3 +- install/helm/agones/defaultfeaturegates.yaml | 3 +- pkg/apis/agones/v1/gameserver.go | 2453 +++++----- pkg/apis/agones/v1/gameserver_test.go | 32 - pkg/cloudproduct/gke/gke.go | 645 ++- pkg/cloudproduct/gke/gke_test.go | 27 +- pkg/gameservers/gameservers_test.go | 4 - pkg/util/runtime/features.go | 562 +-- site/content/en/docs/Guides/feature-stages.md | 2 +- test/e2e/gameserver_test.go | 4019 ++++++++--------- test/upgrade/versionMap.yaml | 8 +- 12 files changed, 3850 insertions(+), 3911 deletions(-) diff --git a/build/Makefile b/build/Makefile index ef5e125c3a..f3a32dfd9c 100644 --- a/build/Makefile +++ b/build/Makefile @@ -69,7 +69,7 @@ KIND_CONTAINER_NAME=$(KIND_PROFILE)-control-plane GS_TEST_IMAGE ?= us-docker.pkg.dev/agones-images/examples/simple-game-server:0.43 # Enable all beta feature gates. Keep in sync with `true` (beta) entries in pkg/util/runtime/features.go:featureDefaults -BETA_FEATURE_GATES ?= "CountsAndLists=true&GKEAutopilotExtendedDurationPods=true&PortPolicyNone=true&PortRanges=true&RollingUpdateFix=true&ScheduledAutoscaler=true&SidecarContainers=true&FleetAutoscaleRequestMetaData=true" +BETA_FEATURE_GATES ?= "CountsAndLists=true&GKEAutopilotExtendedDurationPods=true&PortRanges=true&RollingUpdateFix=true&ScheduledAutoscaler=true&SidecarContainers=true&FleetAutoscaleRequestMetaData=true" # Enable all alpha feature gates. Keep in sync with `false` (alpha) entries in pkg/util/runtime/features.go:featureDefaults @@ -998,3 +998,4 @@ install-release: $(ensure-build-image) # Validates the Helm values.yaml file against the values.schema.json file. helm-lint: $(DOCKER_RUN) bash -c 'cd $(mount_path)/install/helm/agones && helm lint .' + diff --git a/cloudbuild.yaml b/cloudbuild.yaml index bb7c01740a..13abc7e052 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -322,7 +322,7 @@ steps: # Keep in sync with the inverse of 'alpha' and 'beta' features in # pkg/util/runtime/features.go:featureDefaults - featureWithGate="PlayerAllocationFilter=true&FleetAutoscaleRequestMetaData=false&PlayerTracking=true&CountsAndLists=false&RollingUpdateFix=false&PortRanges=false&PortPolicyNone=false&ScheduledAutoscaler=false&GKEAutopilotExtendedDurationPods=false&SidecarContainers=false&WasmAutoscaler=true&Example=true" + featureWithGate="PlayerAllocationFilter=true&FleetAutoscaleRequestMetaData=false&PlayerTracking=true&CountsAndLists=false&RollingUpdateFix=false&PortRanges=false&ScheduledAutoscaler=false&GKEAutopilotExtendedDurationPods=false&SidecarContainers=false&WasmAutoscaler=true&Example=true" featureWithoutGate="" # Use this if specific feature gates can only be supported on specific Kubernetes versions. @@ -464,3 +464,4 @@ logsBucket: gs://agones-build-logs options: machineType: E2_HIGHCPU_32 dynamic_substitutions: true + diff --git a/install/helm/agones/defaultfeaturegates.yaml b/install/helm/agones/defaultfeaturegates.yaml index 9aab1d8e35..91d85b4c9f 100644 --- a/install/helm/agones/defaultfeaturegates.yaml +++ b/install/helm/agones/defaultfeaturegates.yaml @@ -19,12 +19,12 @@ # previous version with the feature flag do not fail on parsing an unknown flag. DisableResyncOnSDKServer: true AutopilotPassthroughPort: true +PortPolicyNone: true # Beta features CountsAndLists: true GKEAutopilotExtendedDurationPods: true -PortPolicyNone: true PortRanges: true ScheduledAutoscaler: true RollingUpdateFix: true @@ -41,3 +41,4 @@ ProcessorAllocator: false # Example feature Example: false + diff --git a/pkg/apis/agones/v1/gameserver.go b/pkg/apis/agones/v1/gameserver.go index fc94ec99d2..50c5f1c422 100644 --- a/pkg/apis/agones/v1/gameserver.go +++ b/pkg/apis/agones/v1/gameserver.go @@ -1,1230 +1,1223 @@ -// Copyright Contributors to Agones a Series of LF Projects, LLC. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package v1 - -import ( - "encoding/json" - "fmt" - "net" - "slices" - "strings" - - "agones.dev/agones/pkg" - "agones.dev/agones/pkg/apis" - "agones.dev/agones/pkg/apis/agones" - "agones.dev/agones/pkg/util/runtime" - "github.com/pkg/errors" - "gomodules.xyz/jsonpatch/v2" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - apimachineryvalidation "k8s.io/apimachinery/pkg/api/validation" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/validation/field" -) - -// GameServerState is the state for the GameServer -type GameServerState string - -const ( - // ListMaxCapacity is the maximum capacity for List in the gamerserver spec and status CRDs. - ListMaxCapacity = int64(1000) -) - -const ( - // GameServerStatePortAllocation is for when a dynamically allocating GameServer - // is being created, an open port needs to be allocated - GameServerStatePortAllocation GameServerState = "PortAllocation" - // GameServerStateCreating is before the Pod for the GameServer is being created - GameServerStateCreating GameServerState = "Creating" - // GameServerStateStarting is for when the Pods for the GameServer are being - // created but are not yet Scheduled - GameServerStateStarting GameServerState = "Starting" - // GameServerStateScheduled is for when we have determined that the Pod has been - // scheduled in the cluster -- basically, we have a NodeName - GameServerStateScheduled GameServerState = "Scheduled" - // GameServerStateRequestReady is when the GameServer has declared that it is ready - GameServerStateRequestReady GameServerState = "RequestReady" - // GameServerStateReady is when a GameServer is ready to take connections - // from Game clients - GameServerStateReady GameServerState = "Ready" - // GameServerStateShutdown is when the GameServer has shutdown and everything needs to be - // deleted from the cluster - GameServerStateShutdown GameServerState = "Shutdown" - // GameServerStateError is when something has gone wrong with the Gameserver and - // it cannot be resolved - GameServerStateError GameServerState = "Error" - // GameServerStateUnhealthy is when the GameServer has failed its health checks - GameServerStateUnhealthy GameServerState = "Unhealthy" - // GameServerStateReserved is for when a GameServer is reserved and therefore can be allocated but not removed - GameServerStateReserved GameServerState = "Reserved" - // GameServerStateAllocated is when the GameServer has been allocated to a session - GameServerStateAllocated GameServerState = "Allocated" -) - -// PortPolicy is the port policy for the GameServer -type PortPolicy string - -const ( - // Static PortPolicy means that the user defines the hostPort to be used - // in the configuration. - Static PortPolicy = "Static" - // Dynamic PortPolicy means that the system will choose an open - // port for the GameServer in question - Dynamic PortPolicy = "Dynamic" - // Passthrough dynamically sets the `containerPort` to the same value as the dynamically selected hostPort. - // This will mean that users will need to lookup what port has been opened through the server side SDK. - Passthrough PortPolicy = "Passthrough" - // None means the `hostPort` is ignored and if defined, the `containerPort` (optional) is used to set the port on the GameServer instance. - None PortPolicy = "None" -) - -// EvictionSafe specified whether the game server supports termination via SIGTERM -type EvictionSafe string - -const ( - // EvictionSafeAlways means the game server supports termination via SIGTERM, and wants eviction signals - // from Cluster Autoscaler scaledown and node upgrades. - EvictionSafeAlways EvictionSafe = "Always" - // EvictionSafeOnUpgrade means the game server supports termination via SIGTERM, and wants eviction signals - // from node upgrades, but not Cluster Autoscaler scaledown. - EvictionSafeOnUpgrade EvictionSafe = "OnUpgrade" - // EvictionSafeNever means the game server should run to completion and may not understand SIGTERM. Eviction - // from ClusterAutoscaler and upgrades should both be blocked. - EvictionSafeNever EvictionSafe = "Never" -) - -// SdkServerLogLevel is the log level for SDK server (sidecar) logs -type SdkServerLogLevel string - -const ( - // SdkServerLogLevelInfo will cause the SDK server to output all messages except for debug messages. - SdkServerLogLevelInfo SdkServerLogLevel = "Info" - // SdkServerLogLevelDebug will cause the SDK server to output all messages including debug messages. - SdkServerLogLevelDebug SdkServerLogLevel = "Debug" - // SdkServerLogLevelError will cause the SDK server to only output error messages. - SdkServerLogLevelError SdkServerLogLevel = "Error" - // SdkServerLogLevelTrace will cause the SDK server to output all messages, including detailed tracing information. - SdkServerLogLevelTrace SdkServerLogLevel = "Trace" -) - -const ( - // ProtocolTCPUDP Protocol exposes the hostPort allocated for this container for both TCP and UDP. - ProtocolTCPUDP corev1.Protocol = "TCPUDP" - - // DefaultPortRange is the name of the default port range. - DefaultPortRange = "default" - - // RoleLabel is the label in which the Agones role is specified. - // Pods from a GameServer will have the value "gameserver" - RoleLabel = agones.GroupName + "/role" - // GameServerLabelRole is the GameServer label value for RoleLabel - GameServerLabelRole = "gameserver" - // GameServerPodLabel is the label that the name of the GameServer - // is set on the Pod the GameServer controls - GameServerPodLabel = agones.GroupName + "/gameserver" - // GameServerPortPolicyPodLabel is the label to identify the port policy - // of the pod - GameServerPortPolicyPodLabel = agones.GroupName + "/port" - // GameServerContainerAnnotation is the annotation that stores - // which container is the container that runs the dedicated game server - GameServerContainerAnnotation = agones.GroupName + "/container" - // DevAddressAnnotation is an annotation to indicate that a GameServer hosted outside of Agones. - // A locally hosted GameServer is not managed by Agones it is just simply registered. - DevAddressAnnotation = "agones.dev/dev-address" - // GameServerReadyContainerIDAnnotation is an annotation that is set on the GameServer - // becomes ready, so we can track when restarts should occur and when a GameServer - // should be moved to Unhealthy. - GameServerReadyContainerIDAnnotation = agones.GroupName + "/ready-container-id" - // PodSafeToEvictAnnotation is an annotation that the Kubernetes cluster autoscaler uses to - // determine if a pod can safely be evicted to compact a cluster by moving pods between nodes - // and scaling down nodes. - PodSafeToEvictAnnotation = "cluster-autoscaler.kubernetes.io/safe-to-evict" - // SafeToEvictLabel is a label that, when "false", matches the restrictive PDB agones-gameserver-safe-to-evict-false. - SafeToEvictLabel = agones.GroupName + "/safe-to-evict" - // GameServerErroredAtAnnotation is an annotation that records the timestamp the GameServer entered the - // error state. The timestamp is encoded in RFC3339 format. - GameServerErroredAtAnnotation = agones.GroupName + "/errored-at" - // FinalizerName is the domain name and finalizer path used to manage garbage collection of the GameServer. - FinalizerName = agones.GroupName + "/controller" - - // NodePodIP identifies an IP address from a pod. - NodePodIP corev1.NodeAddressType = "PodIP" - - // PassthroughPortAssignmentAnnotation is an annotation to keep track of game server container and its Passthrough ports indices - PassthroughPortAssignmentAnnotation = "agones.dev/container-passthrough-port-assignment" - - // True is the string "true" to appease the goconst lint. - True = "true" - // False is the string "false" to appease the goconst lint. - False = "false" -) - -var ( - // GameServerRolePodSelector is the selector to get all GameServer Pods - GameServerRolePodSelector = labels.SelectorFromSet(labels.Set{RoleLabel: GameServerLabelRole}) - - // TerminalGameServerStates is a set (map[GameServerState]bool) of states from which a GameServer will not recover. - // From state diagram at https://agones.dev/site/docs/reference/gameserver/ - TerminalGameServerStates = map[GameServerState]bool{ - GameServerStateShutdown: true, - GameServerStateError: true, - GameServerStateUnhealthy: true, - } -) - -// +genclient -// +genclient:noStatus -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// GameServer is the data structure for a GameServer resource. -// It is worth noting that while there is a `GameServerStatus` Status entry for the `GameServer`, it is not -// defined as a subresource - unlike `Fleet` and other Agones resources. -// This is so that we can retain the ability to change multiple aspects of a `GameServer` in a single atomic operation, -// which is particularly useful for operations such as allocation. -type GameServer struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec GameServerSpec `json:"spec"` - Status GameServerStatus `json:"status"` -} - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// GameServerList is a list of GameServer resources -type GameServerList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - - Items []GameServer `json:"items"` -} - -// GameServerTemplateSpec is a template for GameServers -type GameServerTemplateSpec struct { - metav1.ObjectMeta `json:"metadata,omitempty"` - Spec GameServerSpec `json:"spec"` -} - -// GameServerSpec is the spec for a GameServer resource -type GameServerSpec struct { - // Container specifies which Pod container is the game server. Only required if there is more than one - // container defined - Container string `json:"container,omitempty"` - // Ports are the array of ports that can be exposed via the game server - Ports []GameServerPort `json:"ports,omitempty"` - // Health configures health checking - Health Health `json:"health,omitempty"` - // Scheduling strategy. Defaults to "Packed" - Scheduling apis.SchedulingStrategy `json:"scheduling,omitempty"` - // SdkServer specifies parameters for the Agones SDK Server sidecar container - SdkServer SdkServer `json:"sdkServer,omitempty"` - // Template describes the Pod that will be created for the GameServer - Template corev1.PodTemplateSpec `json:"template"` - // (Alpha, PlayerTracking feature flag) Players provides the configuration for player tracking features. - // +optional - Players *PlayersSpec `json:"players,omitempty"` - // (Beta, CountsAndLists feature flag) Counters provides the configuration for tracking of int64 values against a GameServer. - // Keys must be declared at GameServer creation time. - // +optional - Counters map[string]CounterStatus `json:"counters,omitempty"` - // (Beta, CountsAndLists feature flag) Lists provides the configuration for tracking of lists of up to 1000 values against a GameServer. - // Keys must be declared at GameServer creation time. - // +optional - Lists map[string]ListStatus `json:"lists,omitempty"` - // Eviction specifies the eviction tolerance of the GameServer. Defaults to "Never". - // +optional - Eviction *Eviction `json:"eviction,omitempty"` - // immutableReplicas is present in gameservers.agones.dev but omitted here (it's always 1). -} - -// PlayersSpec tracks the initial player capacity -type PlayersSpec struct { - InitialCapacity int64 `json:"initialCapacity,omitempty"` -} - -// Eviction specifies the eviction tolerance of the GameServer -type Eviction struct { - // Game server supports termination via SIGTERM: - // - Always: Allow eviction for both Cluster Autoscaler and node drain for upgrades - // - OnUpgrade: Allow eviction for upgrades alone - // - Never (default): Pod should run to completion - Safe EvictionSafe `json:"safe,omitempty"` -} - -// Health configures health checking on the GameServer -type Health struct { - // Disabled is whether health checking is disabled or not - Disabled bool `json:"disabled,omitempty"` - // PeriodSeconds is the number of seconds each health ping has to occur in - PeriodSeconds int32 `json:"periodSeconds,omitempty"` - // FailureThreshold how many failures in a row constitutes unhealthy - FailureThreshold int32 `json:"failureThreshold,omitempty"` - // InitialDelaySeconds initial delay before checking health - InitialDelaySeconds int32 `json:"initialDelaySeconds,omitempty"` -} - -// GameServerPort defines a set of Ports that -// are to be exposed via the GameServer -type GameServerPort struct { - // Name is the descriptive name of the port - Name string `json:"name,omitempty"` - // (Alpha, PortRanges feature flag) Range is the port range name from which to select a port when using a - // 'Dynamic' or 'Passthrough' port policy. - // +optional - Range string `json:"range,omitempty"` - // PortPolicy defines the policy for how the HostPort is populated. - // Dynamic port will allocate a HostPort within the selected MIN_PORT and MAX_PORT range passed to the controller - // at installation time. - // When `Static` portPolicy is specified, `HostPort` is required, to specify the port that game clients will - // connect to - // `Passthrough` dynamically sets the `containerPort` to the same value as the dynamically selected hostPort. - // `None` portPolicy ignores `HostPort` and the `containerPort` (optional) is used to set the port on the GameServer instance. - PortPolicy PortPolicy `json:"portPolicy,omitempty"` - // Container is the name of the container or sidecar container on which to open the port. Defaults to the game server container. - // +optional - Container *string `json:"container,omitempty"` - // ContainerPort is the port that is being opened on the specified container's process - ContainerPort int32 `json:"containerPort,omitempty"` - // HostPort the port exposed on the host for clients to connect to - HostPort int32 `json:"hostPort,omitempty"` - // Protocol is the network protocol being used. Defaults to UDP. TCP and TCPUDP are other options. - Protocol corev1.Protocol `json:"protocol,omitempty"` -} - -// SdkServer specifies parameters for the Agones SDK Server sidecar container -type SdkServer struct { - // LogLevel for SDK server (sidecar) logs. Defaults to "Info" - LogLevel SdkServerLogLevel `json:"logLevel,omitempty"` - // GRPCPort is the port on which the SDK Server binds the gRPC server to accept incoming connections - GRPCPort int32 `json:"grpcPort,omitempty"` - // HTTPPort is the port on which the SDK Server binds the HTTP gRPC gateway server to accept incoming connections - HTTPPort int32 `json:"httpPort,omitempty"` -} - -// GameServerStatus is the status for a GameServer resource -type GameServerStatus struct { - // GameServerState is the current state of a GameServer, e.g. Creating, Starting, Ready, etc - State GameServerState `json:"state"` - Ports []GameServerStatusPort `json:"ports"` - Address string `json:"address"` - // Addresses is the array of addresses at which the GameServer can be reached; copy of Node.Status.addresses. - // +optional - Addresses []corev1.NodeAddress `json:"addresses"` - NodeName string `json:"nodeName"` - ReservedUntil *metav1.Time `json:"reservedUntil"` - // [Stage:Alpha] - // [FeatureFlag:PlayerTracking] - // +optional - Players *PlayerStatus `json:"players"` - // (Beta, CountsAndLists feature flag) Counters and Lists provides the configuration for generic tracking features. - // +optional - Counters map[string]CounterStatus `json:"counters,omitempty"` - // +optional - Lists map[string]ListStatus `json:"lists,omitempty"` - // Eviction specifies the eviction tolerance of the GameServer. - // +optional - Eviction *Eviction `json:"eviction,omitempty"` - // immutableReplicas is present in gameservers.agones.dev but omitted here (it's always 1). -} - -// GameServerStatusPort shows the port that was allocated to a -// GameServer. -type GameServerStatusPort struct { - Name string `json:"name,omitempty"` - Port int32 `json:"port"` -} - -// PlayerStatus stores the current player capacity values -type PlayerStatus struct { - Count int64 `json:"count"` - Capacity int64 `json:"capacity"` - IDs []string `json:"ids"` -} - -// CounterStatus stores the current counter values and maximum capacity -type CounterStatus struct { - Count int64 `json:"count"` - Capacity int64 `json:"capacity"` -} - -// ListStatus stores the current list values and maximum capacity -type ListStatus struct { - Capacity int64 `json:"capacity"` - Values []string `json:"values"` -} - -// ApplyDefaults applies default values to the GameServer if they are not already populated -func (gs *GameServer) ApplyDefaults() { - // VersionAnnotation is the annotation that stores - // the version of sdk which runs in a sidecar - if gs.ObjectMeta.Annotations == nil { - gs.ObjectMeta.Annotations = map[string]string{} - } - gs.ObjectMeta.Annotations[VersionAnnotation] = pkg.Version - gs.ObjectMeta.Finalizers = append(gs.ObjectMeta.Finalizers, FinalizerName) - - gs.Spec.ApplyDefaults() - gs.applyStatusDefaults() -} - -// ApplyDefaults applies default values to the GameServerSpec if they are not already populated -func (gss *GameServerSpec) ApplyDefaults() { - gss.applyContainerDefaults() - gss.applyPortDefaults() - gss.applyHealthDefaults() - gss.applyEvictionDefaults() - gss.applySchedulingDefaults() - gss.applySdkServerDefaults() -} - -// applySdkServerDefaults applies the default log level ("Info") for the sidecar -func (gss *GameServerSpec) applySdkServerDefaults() { - if gss.SdkServer.LogLevel == "" { - gss.SdkServer.LogLevel = SdkServerLogLevelInfo - } - if gss.SdkServer.GRPCPort == 0 { - gss.SdkServer.GRPCPort = 9357 - } - if gss.SdkServer.HTTPPort == 0 { - gss.SdkServer.HTTPPort = 9358 - } -} - -// applyContainerDefaults applies the container defaults -func (gss *GameServerSpec) applyContainerDefaults() { - if len(gss.Template.Spec.Containers) == 1 { - gss.Container = gss.Template.Spec.Containers[0].Name - } -} - -// applyHealthDefaults applies health checking defaults -func (gss *GameServerSpec) applyHealthDefaults() { - if !gss.Health.Disabled { - if gss.Health.PeriodSeconds <= 0 { - gss.Health.PeriodSeconds = 5 - } - if gss.Health.FailureThreshold <= 0 { - gss.Health.FailureThreshold = 3 - } - if gss.Health.InitialDelaySeconds <= 0 { - gss.Health.InitialDelaySeconds = 5 - } - } -} - -// applyStatusDefaults applies Status defaults -func (gs *GameServer) applyStatusDefaults() { - if gs.Status.State == "" { - gs.Status.State = GameServerStateCreating - // applyStatusDefaults() should be called after applyPortDefaults() - if gs.HasPortPolicy(Dynamic) || gs.HasPortPolicy(Passthrough) { - gs.Status.State = GameServerStatePortAllocation - } - } - - if runtime.FeatureEnabled(runtime.FeaturePlayerTracking) { - // set value if enabled, otherwise very easy to accidentally panic - // when gs.Status.Players is nil - if gs.Status.Players == nil { - gs.Status.Players = &PlayerStatus{} - } - if gs.Spec.Players != nil { - gs.Status.Players.Capacity = gs.Spec.Players.InitialCapacity - } - } - - gs.applyEvictionStatus() - gs.applyCountsListsStatus() -} - -// applyPortDefaults applies default values for all ports -func (gss *GameServerSpec) applyPortDefaults() { - for i, p := range gss.Ports { - // basic spec - if p.PortPolicy == "" { - gss.Ports[i].PortPolicy = Dynamic - } - - if p.Range == "" { - gss.Ports[i].Range = DefaultPortRange - } - - if p.Protocol == "" { - gss.Ports[i].Protocol = "UDP" - } - - if p.Container == nil || *p.Container == "" { - gss.Ports[i].Container = &gss.Container - } - } -} - -func (gss *GameServerSpec) applySchedulingDefaults() { - if gss.Scheduling == "" { - gss.Scheduling = apis.Packed - } -} - -func (gss *GameServerSpec) applyEvictionDefaults() { - if gss.Eviction == nil { - gss.Eviction = &Eviction{} - } - if gss.Eviction.Safe == "" { - gss.Eviction.Safe = EvictionSafeNever - } -} - -func (gs *GameServer) applyEvictionStatus() { - gs.Status.Eviction = gs.Spec.Eviction.DeepCopy() - if gs.Spec.Template.ObjectMeta.Annotations[PodSafeToEvictAnnotation] == "true" { - if gs.Status.Eviction == nil { - gs.Status.Eviction = &Eviction{} - } - gs.Status.Eviction.Safe = EvictionSafeAlways - } -} - -func (gs *GameServer) applyCountsListsStatus() { - if !runtime.FeatureEnabled(runtime.FeatureCountsAndLists) { - return - } - if gs.Spec.Counters != nil { - countersCopy := make(map[string]CounterStatus, len(gs.Spec.Counters)) - for key, val := range gs.Spec.Counters { - countersCopy[key] = *val.DeepCopy() - } - gs.Status.Counters = countersCopy - } - if gs.Spec.Lists != nil { - listsCopy := make(map[string]ListStatus, len(gs.Spec.Lists)) - for key, val := range gs.Spec.Lists { - listsCopy[key] = *val.DeepCopy() - } - gs.Status.Lists = listsCopy - } -} - -// validateFeatureGates checks if fields are set when the associated feature gate is not set. -func (gss *GameServerSpec) validateFeatureGates(fldPath *field.Path) field.ErrorList { - var allErrs field.ErrorList - if !runtime.FeatureEnabled(runtime.FeaturePlayerTracking) { - if gss.Players != nil { - allErrs = append(allErrs, field.Forbidden(fldPath.Child("players"), fmt.Sprintf("Value cannot be set unless feature flag %s is enabled", runtime.FeaturePlayerTracking))) - } - } - - if !runtime.FeatureEnabled(runtime.FeatureCountsAndLists) { - if gss.Counters != nil { - allErrs = append(allErrs, field.Forbidden(fldPath.Child("counters"), fmt.Sprintf("Value cannot be set unless feature flag %s is enabled", runtime.FeatureCountsAndLists))) - } - if gss.Lists != nil { - allErrs = append(allErrs, field.Forbidden(fldPath.Child("lists"), fmt.Sprintf("Value cannot be set unless feature flag %s is enabled", runtime.FeatureCountsAndLists))) - } - } - - if !runtime.FeatureEnabled(runtime.FeaturePortPolicyNone) { - for i, p := range gss.Ports { - if p.PortPolicy == None { - allErrs = append(allErrs, field.Forbidden(fldPath.Child("ports").Index(i).Child("portPolicy"), fmt.Sprintf("Value cannot be set to %s unless feature flag %s is enabled", None, runtime.FeaturePortPolicyNone))) - } - } - } - - return allErrs -} - -// Validate validates the GameServerSpec configuration. -// devAddress is a specific IP address used for local Gameservers, for fleets "" is used -// If a GameServer Spec is invalid there will be > 0 values in the returned array -func (gss *GameServerSpec) Validate(apiHooks APIHooks, devAddress string, fldPath *field.Path) field.ErrorList { - allErrs := gss.validateFeatureGates(fldPath) - if len(devAddress) > 0 { - // verify that the value is a valid IP address. - if net.ParseIP(devAddress) == nil { - // Authentication is only required if the gameserver is created directly. - allErrs = append(allErrs, field.Invalid(field.NewPath("metadata", "annotations", DevAddressAnnotation), devAddress, "must be a valid IP address")) - } - - for i, p := range gss.Ports { - if p.HostPort == 0 { - allErrs = append(allErrs, field.Required(fldPath.Child("ports").Index(i).Child("hostPort"), DevAddressAnnotation)) - } - if p.PortPolicy != Static { - allErrs = append(allErrs, field.Required(fldPath.Child("ports").Index(i).Child("portPolicy"), ErrPortPolicyStatic)) - } - } - - allErrs = append(allErrs, validateObjectMeta(&gss.Template.ObjectMeta, fldPath.Child("template", "metadata"))...) - return allErrs - } - - // make sure a name is specified when there is multiple containers in the pod. - if gss.Container == "" && len(gss.Template.Spec.Containers) > 1 { - allErrs = append(allErrs, field.Required(fldPath.Child("container"), ErrContainerRequired)) - } - - // make sure the container value points to a valid container - if !gss.HasContainer(gss.Container, false) { - allErrs = append(allErrs, - field.Invalid( - fldPath.Child("container"), - gss.Container, - "Could not find a container named "+gss.Container, - ), - ) - } - - // no host port when using dynamic PortPolicy - for i, p := range gss.Ports { - path := fldPath.Child("ports").Index(i) - if p.PortPolicy == Dynamic || p.PortPolicy == Static { - if p.ContainerPort <= 0 { - allErrs = append(allErrs, field.Required(path.Child("containerPort"), ErrContainerPortRequired)) - } - } - - if p.PortPolicy == Passthrough && p.ContainerPort > 0 { - allErrs = append(allErrs, field.Required(path.Child("containerPort"), ErrContainerPortPassthrough)) - } - - if p.HostPort > 0 && (p.PortPolicy == Dynamic || p.PortPolicy == Passthrough) { - allErrs = append(allErrs, field.Forbidden(path.Child("hostPort"), ErrHostPort)) - } - - if p.Container != nil && gss.Container != "" { - if !gss.HasContainer(*p.Container, true) { - allErrs = append(allErrs, field.Invalid(path.Child("container"), *p.Container, ErrContainerNameInvalid)) - } - } - } - for i, c := range gss.Template.Spec.InitContainers { - path := fldPath.Child("template", "spec", "initContainers").Index(i) - allErrs = append(allErrs, ValidateResourceRequirements(&c.Resources, path.Child("resources"))...) - } - for i, c := range gss.Template.Spec.Containers { - path := fldPath.Child("template", "spec", "containers").Index(i) - allErrs = append(allErrs, ValidateResourceRequirements(&c.Resources, path.Child("resources"))...) - } - - allErrs = append(allErrs, apiHooks.ValidateGameServerSpec(gss, fldPath)...) - allErrs = append(allErrs, validateObjectMeta(&gss.Template.ObjectMeta, fldPath.Child("template", "metadata"))...) - return allErrs -} - -// ValidateResourceRequirements Validates resource requirement spec. -func ValidateResourceRequirements(requirements *corev1.ResourceRequirements, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - limPath := fldPath.Child("limits") - reqPath := fldPath.Child("requests") - - for resourceName, quantity := range requirements.Limits { - fldPath := limPath.Key(string(resourceName)) - // Validate resource quantity. - allErrs = append(allErrs, ValidateNonnegativeQuantity(quantity, fldPath)...) - - } - - for resourceName, quantity := range requirements.Requests { - fldPath := reqPath.Key(string(resourceName)) - // Validate resource quantity. - allErrs = append(allErrs, ValidateNonnegativeQuantity(quantity, fldPath)...) - - // Check that request <= limit. - limitQuantity, exists := requirements.Limits[resourceName] - if exists && quantity.Cmp(limitQuantity) > 0 { - allErrs = append(allErrs, field.Invalid(reqPath, quantity.String(), fmt.Sprintf("must be less than or equal to %s limit of %s", resourceName, limitQuantity.String()))) - } - } - return allErrs -} - -// ValidateNonnegativeQuantity Validates that a Quantity is not negative -func ValidateNonnegativeQuantity(value resource.Quantity, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - if value.Cmp(resource.Quantity{}) < 0 { - allErrs = append(allErrs, field.Invalid(fldPath, value.String(), apimachineryvalidation.IsNegativeErrorMsg)) - } - return allErrs -} - -// Validate validates the GameServer configuration. -// If a GameServer is invalid there will be > 0 values in -// the returned array -func (gs *GameServer) Validate(apiHooks APIHooks) field.ErrorList { - allErrs := validateName(gs, field.NewPath("metadata")) - - // make sure the host port is specified if this is a development server - devAddress, _ := gs.GetDevAddress() - allErrs = append(allErrs, gs.Spec.Validate(apiHooks, devAddress, field.NewPath("spec"))...) - return allErrs -} - -// GetDevAddress returns the address for game server. -func (gs *GameServer) GetDevAddress() (string, bool) { - devAddress, hasDevAddress := gs.ObjectMeta.Annotations[DevAddressAnnotation] - return devAddress, hasDevAddress -} - -// IsDeletable returns false if the server is currently allocated/reserved and is not already in the -// process of being deleted -func (gs *GameServer) IsDeletable() bool { - if gs.Status.State == GameServerStateAllocated || gs.Status.State == GameServerStateReserved { - return !gs.ObjectMeta.DeletionTimestamp.IsZero() - } - - return true -} - -// IsBeingDeleted returns true if the server is in the process of being deleted. -func (gs *GameServer) IsBeingDeleted() bool { - return !gs.ObjectMeta.DeletionTimestamp.IsZero() || gs.Status.State == GameServerStateShutdown -} - -// IsBeforeReady returns true if the GameServer Status has yet to move to or past the Ready -// state in its lifecycle, such as Allocated or Reserved, or any of the Error/Unhealthy states -func (gs *GameServer) IsBeforeReady() bool { - switch gs.Status.State { - case GameServerStatePortAllocation: - return true - case GameServerStateCreating: - return true - case GameServerStateStarting: - return true - case GameServerStateScheduled: - return true - case GameServerStateRequestReady: - return true - } - - return false -} - -// IsActive returns true if the GameServer status is Ready, Reserved, or Allocated state. -func (gs *GameServer) IsActive() bool { - switch gs.Status.State { - case GameServerStateAllocated: - return true - case GameServerStateReady: - return true - case GameServerStateReserved: - return true - } - - return false -} - -// HasContainer determines if the GameServerSpec has a container with the specified name. -// Init containers with RestartPolicy `Always` will be considered if `includeSidecar` is true. -func (gss *GameServerSpec) HasContainer(name string, includeSidecar bool) bool { - for _, c := range gss.Template.Spec.Containers { - if c.Name == name { - return true - } - } - if !includeSidecar { - return false - } - - for _, c := range gss.Template.Spec.InitContainers { - if c.RestartPolicy == nil || *c.RestartPolicy != corev1.ContainerRestartPolicyAlways { - continue - } - if c.Name == name { - return true - } - } - return false -} - -// ApplyToPodContainer applies func(v1.Container) to the specified container in the pod. -// Init containers with RestartPolicy Always will also be considered. -// Returns an error if the container is not found. -func (gs *GameServer) ApplyToPodContainer(pod *corev1.Pod, containerName string, f func(corev1.Container) corev1.Container) error { - for i, c := range pod.Spec.Containers { - if c.Name == containerName { - pod.Spec.Containers[i] = f(c) - return nil - } - } - for i, c := range pod.Spec.InitContainers { - if c.RestartPolicy == nil || *c.RestartPolicy != corev1.ContainerRestartPolicyAlways { - continue - } - if c.Name == containerName { - pod.Spec.InitContainers[i] = f(c) - return nil - } - } - return errors.Errorf("failed to find container named %s in pod spec", containerName) -} - -// Pod creates a new Pod from the PodTemplateSpec -// attached to the GameServer resource -func (gs *GameServer) Pod(apiHooks APIHooks, sidecars ...corev1.Container) (*corev1.Pod, error) { - pod := &corev1.Pod{ - ObjectMeta: *gs.Spec.Template.ObjectMeta.DeepCopy(), - Spec: *gs.Spec.Template.Spec.DeepCopy(), - } - - if len(pod.Spec.Hostname) == 0 { - // replace . with - since it must match RFC 1123 - pod.Spec.Hostname = strings.ReplaceAll(gs.ObjectMeta.Name, ".", "-") - } - - gs.podObjectMeta(pod) - - passthroughContainerPortMap := make(map[string][]int) - for _, p := range gs.Spec.Ports { - var hostPort int32 - portIdx := 0 - - if !runtime.FeatureEnabled(runtime.FeaturePortPolicyNone) || p.PortPolicy != None { - hostPort = p.HostPort - } - - cp := corev1.ContainerPort{ - ContainerPort: p.ContainerPort, - HostPort: hostPort, - Protocol: p.Protocol, - } - err := gs.ApplyToPodContainer(pod, *p.Container, func(c corev1.Container) corev1.Container { - portIdx = len(c.Ports) - c.Ports = append(c.Ports, cp) - - return c - }) - if err != nil { - return nil, err - } - if p.PortPolicy == Passthrough { - passthroughContainerPortMap[*p.Container] = append(passthroughContainerPortMap[*p.Container], portIdx) - } - } - - if len(passthroughContainerPortMap) != 0 { - containerToPassthroughMapJSON, err := json.Marshal(passthroughContainerPortMap) - if err != nil { - return nil, err - } - pod.ObjectMeta.Annotations[PassthroughPortAssignmentAnnotation] = string(containerToPassthroughMapJSON) - } - - if runtime.FeatureEnabled(runtime.FeatureSidecarContainers) { - // make sure all sidecars have a restart policy of Always, so they are valid sidecar containers. - // https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/#sidecar-containers-and-pod-lifecycle - always := corev1.ContainerRestartPolicyAlways - for i := range sidecars { - sidecars[i].RestartPolicy = &always - } - - // addSidecarsAsInitContainers puts the sidecars in the initContainers list so that they can have their own independent - // container restart policies. - pod.Spec.InitContainers = slices.Concat(sidecars, pod.Spec.InitContainers) - - // default GameServer container should also be Restart: Never - if len(pod.Spec.RestartPolicy) == 0 { - pod.Spec.RestartPolicy = corev1.RestartPolicyNever - } - } else { - gs.addSidecarsAsContainers(sidecars, pod) - } - - gs.podScheduling(pod) - - if err := apiHooks.MutateGameServerPod(&gs.Spec, pod); err != nil { - return nil, err - } - if err := apiHooks.SetEviction(gs.Status.Eviction, pod); err != nil { - return nil, err - } - - return pod, nil -} - -// addSidecarsAsContainers puts the sidecars at the start of the general list of containers so that the kubelet starts -// them first -func (gs *GameServer) addSidecarsAsContainers(sidecars []corev1.Container, pod *corev1.Pod) { - containers := make([]corev1.Container, 0, len(sidecars)+len(pod.Spec.Containers)) - containers = append(containers, sidecars...) - containers = append(containers, pod.Spec.Containers...) - pod.Spec.Containers = containers -} - -// podObjectMeta configures the pod ObjectMeta details -func (gs *GameServer) podObjectMeta(pod *corev1.Pod) { - pod.ObjectMeta.GenerateName = "" - // Pods inherit the name of their gameserver. It's safe since there's - // a guarantee that pod won't outlive its parent. - pod.ObjectMeta.Name = gs.ObjectMeta.Name - // Pods for GameServers need to stay in the same namespace - pod.ObjectMeta.Namespace = gs.ObjectMeta.Namespace - // Make sure these are blank, just in case - pod.ObjectMeta.ResourceVersion = "" - pod.ObjectMeta.UID = "" - if pod.ObjectMeta.Labels == nil { - pod.ObjectMeta.Labels = make(map[string]string, 2) - } - if pod.ObjectMeta.Annotations == nil { - pod.ObjectMeta.Annotations = make(map[string]string, 2) - } - pod.ObjectMeta.Labels[RoleLabel] = GameServerLabelRole - // store the GameServer name as a label, for easy lookup later on - pod.ObjectMeta.Labels[GameServerPodLabel] = gs.ObjectMeta.Name - // store the GameServer container as an annotation, to make lookup at a Pod level easier - pod.ObjectMeta.Annotations[GameServerContainerAnnotation] = gs.Spec.Container - ref := metav1.NewControllerRef(gs, SchemeGroupVersion.WithKind("GameServer")) - pod.ObjectMeta.OwnerReferences = append(pod.ObjectMeta.OwnerReferences, *ref) - - // Add Agones version into Pod Annotations - pod.ObjectMeta.Annotations[VersionAnnotation] = pkg.Version -} - -// podScheduling applies the Fleet scheduling strategy to the passed in Pod -// this sets the a PreferredDuringSchedulingIgnoredDuringExecution for GameServer -// pods to a host topology. Basically doing a half decent job of packing GameServer -// pods together. -func (gs *GameServer) podScheduling(pod *corev1.Pod) { - if gs.Spec.Scheduling == apis.Packed { - if pod.Spec.Affinity == nil { - pod.Spec.Affinity = &corev1.Affinity{} - } - if pod.Spec.Affinity.PodAffinity == nil { - pod.Spec.Affinity.PodAffinity = &corev1.PodAffinity{} - } - - wpat := corev1.WeightedPodAffinityTerm{ - Weight: 100, - PodAffinityTerm: corev1.PodAffinityTerm{ - TopologyKey: "kubernetes.io/hostname", - LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{RoleLabel: GameServerLabelRole}}, - }, - } - - pod.Spec.Affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append(pod.Spec.Affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution, wpat) - } -} - -// DisableServiceAccount disables the service account for the gameserver container -func (gs *GameServer) DisableServiceAccount(pod *corev1.Pod) error { - // gameservers don't get access to the k8s api. - emptyVol := corev1.Volume{Name: "empty", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}} - pod.Spec.Volumes = append(pod.Spec.Volumes, emptyVol) - mount := corev1.VolumeMount{MountPath: "/var/run/secrets/kubernetes.io/serviceaccount", Name: emptyVol.Name, ReadOnly: true} - - return gs.ApplyToPodContainer(pod, gs.Spec.Container, func(c corev1.Container) corev1.Container { - c.VolumeMounts = append(c.VolumeMounts, mount) - - return c - }) -} - -// HasPortPolicy checks if there is a port with a given -// PortPolicy -func (gs *GameServer) HasPortPolicy(policy PortPolicy) bool { - for _, p := range gs.Spec.Ports { - if p.PortPolicy == policy { - return true - } - } - return false -} - -// Status returns a GameServerStatusPort for this GameServerPort -func (p GameServerPort) Status() GameServerStatusPort { - if runtime.FeatureEnabled(runtime.FeaturePortPolicyNone) && p.PortPolicy == None { - return GameServerStatusPort{Name: p.Name, Port: p.ContainerPort} - } - - return GameServerStatusPort{Name: p.Name, Port: p.HostPort} -} - -// CountPorts returns the number of -// ports that match condition function -func (gs *GameServer) CountPorts(f func(policy PortPolicy) bool) int { - count := 0 - for _, p := range gs.Spec.Ports { - if f(p.PortPolicy) { - count++ - } - } - return count -} - -// CountPortsForRange returns the number of ports that match condition function and range name. -func (gs *GameServer) CountPortsForRange(name string, f func(policy PortPolicy) bool) int { - count := 0 - for _, p := range gs.Spec.Ports { - if p.Range == name && f(p.PortPolicy) { - count++ - } - } - return count -} - -// Patch creates a JSONPatch to move the current GameServer to the passed in delta GameServer. -// Returned Patch includes a "test" operation that will cause the GameServers.Patch() operation to -// fail if the Game Server has been updated (ResourceVersion has changed) in between when the Patch -// was created and applied. -func (gs *GameServer) Patch(delta *GameServer) ([]byte, error) { - var result []byte - - oldJSON, err := json.Marshal(gs) - if err != nil { - return result, errors.Wrapf(err, "error marshalling to json current GameServer %s", gs.ObjectMeta.Name) - } - - newJSON, err := json.Marshal(delta) - if err != nil { - return result, errors.Wrapf(err, "error marshalling to json delta GameServer %s", delta.ObjectMeta.Name) - } - - patch, err := jsonpatch.CreatePatch(oldJSON, newJSON) - if err != nil { - return result, errors.Wrapf(err, "error creating patch for GameServer %s", gs.ObjectMeta.Name) - } - - // Per https://jsonpatch.com/ "Tests that the specified value is set in the document. If the test - // fails, then the patch as a whole should not apply." - // Used here to check the object has not been updated (has not changed ResourceVersion). - patches := []jsonpatch.JsonPatchOperation{{Operation: "test", Path: "/metadata/resourceVersion", Value: gs.ObjectMeta.ResourceVersion}} - patches = append(patches, patch...) - - result, err = json.Marshal(patches) - return result, errors.Wrapf(err, "error creating json for patch for GameServer %s", gs.ObjectMeta.Name) -} - -// UpdateCount increments or decrements a CounterStatus on a Game Server by the given amount. -func (gs *GameServer) UpdateCount(name string, action string, amount int64) error { - if !(action == GameServerPriorityIncrement || action == GameServerPriorityDecrement) { - return errors.Errorf("unable to UpdateCount with Name %s, Action %s, Amount %d. Allocation action must be one of %s or %s", name, action, amount, GameServerPriorityIncrement, GameServerPriorityDecrement) - } - if amount < 0 { - return errors.Errorf("unable to UpdateCount with Name %s, Action %s, Amount %d. Amount must be greater than 0", name, action, amount) - } - if counter, ok := gs.Status.Counters[name]; ok { - cnt := counter.Count - if action == GameServerPriorityIncrement { - cnt += amount - } else { - cnt -= amount - } - // Truncate to Capacity if Count > Capacity - if cnt > counter.Capacity { - cnt = counter.Capacity - } - // Truncate to Zero if Count is negative - if cnt < 0 { - cnt = 0 - } - counter.Count = cnt - gs.Status.Counters[name] = counter - return nil - } - return errors.Errorf("unable to UpdateCount with Name %s, Action %s, Amount %d. Counter not found in GameServer %s", name, action, amount, gs.ObjectMeta.GetName()) -} - -// UpdateCounterCapacity updates the CounterStatus Capacity to the given capacity. -func (gs *GameServer) UpdateCounterCapacity(name string, capacity int64) error { - if capacity < 0 { - return errors.Errorf("unable to UpdateCounterCapacity: Name %s, Capacity %d. Capacity must be greater than or equal to 0", name, capacity) - } - if counter, ok := gs.Status.Counters[name]; ok { - counter.Capacity = capacity - // If Capacity is now less than Count, reset Count here to equal Capacity - if counter.Count > counter.Capacity { - counter.Count = counter.Capacity - } - gs.Status.Counters[name] = counter - return nil - } - return errors.Errorf("unable to UpdateCounterCapacity: Name %s, Capacity %d. Counter not found in GameServer %s", name, capacity, gs.ObjectMeta.GetName()) -} - -// UpdateListCapacity updates the ListStatus Capacity to the given capacity. -func (gs *GameServer) UpdateListCapacity(name string, capacity int64) error { - - if capacity < 0 || capacity > ListMaxCapacity { - return errors.Errorf("unable to UpdateListCapacity: Name %s, Capacity %d. Capacity must be between 0 and 1000, inclusive", name, capacity) - } - if list, ok := gs.Status.Lists[name]; ok { - list.Capacity = capacity - list.Values = truncateList(list.Capacity, list.Values) - gs.Status.Lists[name] = list - return nil - } - return errors.Errorf("unable to UpdateListCapacity: Name %s, Capacity %d. List not found in GameServer %s", name, capacity, gs.ObjectMeta.GetName()) -} - -// AppendListValues adds unique values to the ListStatus Values list. -func (gs *GameServer) AppendListValues(name string, values []string) error { - if values == nil { - return errors.Errorf("unable to AppendListValues: Name %s, Values %s. Values must not be nil", name, values) - } - if list, ok := gs.Status.Lists[name]; ok { - mergedList := MergeRemoveDuplicates(list.Values, values) - // Any duplicate values are silently dropped. - list.Values = mergedList - list.Values = truncateList(list.Capacity, list.Values) - gs.Status.Lists[name] = list - return nil - } - return errors.Errorf("unable to AppendListValues: Name %s, Values %s. List not found in GameServer %s", name, values, gs.ObjectMeta.GetName()) -} - -// DeleteListValues removes values from the ListStatus Values list. Values in the DeleteListValues -// list that are not in the ListStatus Values list are ignored. -func (gs *GameServer) DeleteListValues(name string, values []string) error { - if values == nil { - return errors.Errorf("unable to DeleteListValues: Name %s, Values %s. Values must not be nil", name, values) - } - if list, ok := gs.Status.Lists[name]; ok { - deleteValuesMap := make(map[string]bool) - for _, value := range values { - deleteValuesMap[value] = true - } - newList := deleteValues(list.Values, deleteValuesMap) - list.Values = newList - gs.Status.Lists[name] = list - return nil - } - return errors.Errorf("unable to DeleteListValues: Name %s, Values %s. List not found in GameServer %s", name, values, gs.ObjectMeta.GetName()) -} - -// deleteValues returns a new list with all the values in valuesList that are not keys in deleteValuesMap. -func deleteValues(valuesList []string, deleteValuesMap map[string]bool) []string { - newValuesList := []string{} - for _, value := range valuesList { - if _, ok := deleteValuesMap[value]; ok { - continue - } - newValuesList = append(newValuesList, value) - } - return newValuesList -} - -// truncateList truncates the list to the given capacity -func truncateList(capacity int64, list []string) []string { - if list == nil || len(list) <= int(capacity) { - return list - } - list = append([]string{}, list[:capacity]...) - return list -} - -// MergeRemoveDuplicates merges two lists and removes any duplicate values. -// Maintains ordering, so new values from list2 are appended to the end of list1. -// Returns a new list with unique values only. -func MergeRemoveDuplicates(list1 []string, list2 []string) []string { - uniqueList := []string{} - listMap := make(map[string]bool) - for _, v1 := range list1 { - if _, ok := listMap[v1]; !ok { - uniqueList = append(uniqueList, v1) - listMap[v1] = true - } - } - for _, v2 := range list2 { - if _, ok := listMap[v2]; !ok { - uniqueList = append(uniqueList, v2) - listMap[v2] = true - } - } - return uniqueList -} - -// CompareCountAndListPriorities compares two game servers based on a list of CountsAndLists Priorities using available -// capacity as the comparison. -func (gs *GameServer) CompareCountAndListPriorities(priorities []Priority, other *GameServer) *bool { - for _, priority := range priorities { - res := gs.compareCountAndListPriority(&priority, other) - if res != nil { - // reverse if descending - if priority.Order == GameServerPriorityDescending { - flip := !*res - return &flip - } - - return res - } - } - - return nil -} - -// compareCountAndListPriority compares two game servers based on a CountsAndLists Priority using available -// capacity (Capacity - Count for Counters, and Capacity - len(Values) for Lists) as the comparison. -// Returns true if gs1 < gs2; false if gs1 > gs2; nil if gs1 == gs2; nil if neither gamer server has the Priority. -// If only one game server has the Priority, prefer that server. I.e. nil < gsX when Priority -// Order is Descending (3, 2, 1, 0, nil), and nil > gsX when Order is Ascending (0, 1, 2, 3, nil). -func (gs *GameServer) compareCountAndListPriority(p *Priority, other *GameServer) *bool { - var gs1ok, gs2ok bool - t := true - f := false - switch p.Type { - case GameServerPriorityCounter: - // Check if both game servers contain the Counter. - counter1, ok1 := gs.Status.Counters[p.Key] - counter2, ok2 := other.Status.Counters[p.Key] - // If both game servers have the Counter - if ok1 && ok2 { - availCapacity1 := counter1.Capacity - counter1.Count - availCapacity2 := counter2.Capacity - counter2.Count - if availCapacity1 < availCapacity2 { - return &t - } - if availCapacity1 > availCapacity2 { - return &f - } - if availCapacity1 == availCapacity2 { - return nil - } - } - gs1ok = ok1 - gs2ok = ok2 - case GameServerPriorityList: - // Check if both game servers contain the List. - list1, ok1 := gs.Status.Lists[p.Key] - list2, ok2 := other.Status.Lists[p.Key] - // If both game servers have the List - if ok1 && ok2 { - availCapacity1 := list1.Capacity - int64(len(list1.Values)) - availCapacity2 := list2.Capacity - int64(len(list2.Values)) - if availCapacity1 < availCapacity2 { - return &t - } - if availCapacity1 > availCapacity2 { - return &f - } - if availCapacity1 == availCapacity2 { - return nil - } - } - gs1ok = ok1 - gs2ok = ok2 - } - // If only one game server has the Priority, prefer that server. I.e. nil < gsX when Order is - // Descending (3, 2, 1, 0, nil), and nil > gsX when Order is Ascending (0, 1, 2, 3, nil). - if (gs1ok && p.Order == GameServerPriorityDescending) || - (gs2ok && p.Order == GameServerPriorityAscending) { - return &f - } - if (gs1ok && p.Order == GameServerPriorityAscending) || - (gs2ok && p.Order == GameServerPriorityDescending) { - return &t - } - // If neither game server has the Priority - return nil -} +// Copyright Contributors to Agones a Series of LF Projects, LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "encoding/json" + "fmt" + "net" + "slices" + "strings" + + "agones.dev/agones/pkg" + "agones.dev/agones/pkg/apis" + "agones.dev/agones/pkg/apis/agones" + "agones.dev/agones/pkg/util/runtime" + "github.com/pkg/errors" + "gomodules.xyz/jsonpatch/v2" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + apimachineryvalidation "k8s.io/apimachinery/pkg/api/validation" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +// GameServerState is the state for the GameServer +type GameServerState string + +const ( + // ListMaxCapacity is the maximum capacity for List in the gamerserver spec and status CRDs. + ListMaxCapacity = int64(1000) +) + +const ( + // GameServerStatePortAllocation is for when a dynamically allocating GameServer + // is being created, an open port needs to be allocated + GameServerStatePortAllocation GameServerState = "PortAllocation" + // GameServerStateCreating is before the Pod for the GameServer is being created + GameServerStateCreating GameServerState = "Creating" + // GameServerStateStarting is for when the Pods for the GameServer are being + // created but are not yet Scheduled + GameServerStateStarting GameServerState = "Starting" + // GameServerStateScheduled is for when we have determined that the Pod has been + // scheduled in the cluster -- basically, we have a NodeName + GameServerStateScheduled GameServerState = "Scheduled" + // GameServerStateRequestReady is when the GameServer has declared that it is ready + GameServerStateRequestReady GameServerState = "RequestReady" + // GameServerStateReady is when a GameServer is ready to take connections + // from Game clients + GameServerStateReady GameServerState = "Ready" + // GameServerStateShutdown is when the GameServer has shutdown and everything needs to be + // deleted from the cluster + GameServerStateShutdown GameServerState = "Shutdown" + // GameServerStateError is when something has gone wrong with the Gameserver and + // it cannot be resolved + GameServerStateError GameServerState = "Error" + // GameServerStateUnhealthy is when the GameServer has failed its health checks + GameServerStateUnhealthy GameServerState = "Unhealthy" + // GameServerStateReserved is for when a GameServer is reserved and therefore can be allocated but not removed + GameServerStateReserved GameServerState = "Reserved" + // GameServerStateAllocated is when the GameServer has been allocated to a session + GameServerStateAllocated GameServerState = "Allocated" +) + +// PortPolicy is the port policy for the GameServer +type PortPolicy string + +const ( + // Static PortPolicy means that the user defines the hostPort to be used + // in the configuration. + Static PortPolicy = "Static" + // Dynamic PortPolicy means that the system will choose an open + // port for the GameServer in question + Dynamic PortPolicy = "Dynamic" + // Passthrough dynamically sets the `containerPort` to the same value as the dynamically selected hostPort. + // This will mean that users will need to lookup what port has been opened through the server side SDK. + Passthrough PortPolicy = "Passthrough" + // None means the `hostPort` is ignored and if defined, the `containerPort` (optional) is used to set the port on the GameServer instance. + None PortPolicy = "None" +) + +// EvictionSafe specified whether the game server supports termination via SIGTERM +type EvictionSafe string + +const ( + // EvictionSafeAlways means the game server supports termination via SIGTERM, and wants eviction signals + // from Cluster Autoscaler scaledown and node upgrades. + EvictionSafeAlways EvictionSafe = "Always" + // EvictionSafeOnUpgrade means the game server supports termination via SIGTERM, and wants eviction signals + // from node upgrades, but not Cluster Autoscaler scaledown. + EvictionSafeOnUpgrade EvictionSafe = "OnUpgrade" + // EvictionSafeNever means the game server should run to completion and may not understand SIGTERM. Eviction + // from ClusterAutoscaler and upgrades should both be blocked. + EvictionSafeNever EvictionSafe = "Never" +) + +// SdkServerLogLevel is the log level for SDK server (sidecar) logs +type SdkServerLogLevel string + +const ( + // SdkServerLogLevelInfo will cause the SDK server to output all messages except for debug messages. + SdkServerLogLevelInfo SdkServerLogLevel = "Info" + // SdkServerLogLevelDebug will cause the SDK server to output all messages including debug messages. + SdkServerLogLevelDebug SdkServerLogLevel = "Debug" + // SdkServerLogLevelError will cause the SDK server to only output error messages. + SdkServerLogLevelError SdkServerLogLevel = "Error" + // SdkServerLogLevelTrace will cause the SDK server to output all messages, including detailed tracing information. + SdkServerLogLevelTrace SdkServerLogLevel = "Trace" +) + +const ( + // ProtocolTCPUDP Protocol exposes the hostPort allocated for this container for both TCP and UDP. + ProtocolTCPUDP corev1.Protocol = "TCPUDP" + + // DefaultPortRange is the name of the default port range. + DefaultPortRange = "default" + + // RoleLabel is the label in which the Agones role is specified. + // Pods from a GameServer will have the value "gameserver" + RoleLabel = agones.GroupName + "/role" + // GameServerLabelRole is the GameServer label value for RoleLabel + GameServerLabelRole = "gameserver" + // GameServerPodLabel is the label that the name of the GameServer + // is set on the Pod the GameServer controls + GameServerPodLabel = agones.GroupName + "/gameserver" + // GameServerPortPolicyPodLabel is the label to identify the port policy + // of the pod + GameServerPortPolicyPodLabel = agones.GroupName + "/port" + // GameServerContainerAnnotation is the annotation that stores + // which container is the container that runs the dedicated game server + GameServerContainerAnnotation = agones.GroupName + "/container" + // DevAddressAnnotation is an annotation to indicate that a GameServer hosted outside of Agones. + // A locally hosted GameServer is not managed by Agones it is just simply registered. + DevAddressAnnotation = "agones.dev/dev-address" + // GameServerReadyContainerIDAnnotation is an annotation that is set on the GameServer + // becomes ready, so we can track when restarts should occur and when a GameServer + // should be moved to Unhealthy. + GameServerReadyContainerIDAnnotation = agones.GroupName + "/ready-container-id" + // PodSafeToEvictAnnotation is an annotation that the Kubernetes cluster autoscaler uses to + // determine if a pod can safely be evicted to compact a cluster by moving pods between nodes + // and scaling down nodes. + PodSafeToEvictAnnotation = "cluster-autoscaler.kubernetes.io/safe-to-evict" + // SafeToEvictLabel is a label that, when "false", matches the restrictive PDB agones-gameserver-safe-to-evict-false. + SafeToEvictLabel = agones.GroupName + "/safe-to-evict" + // GameServerErroredAtAnnotation is an annotation that records the timestamp the GameServer entered the + // error state. The timestamp is encoded in RFC3339 format. + GameServerErroredAtAnnotation = agones.GroupName + "/errored-at" + // FinalizerName is the domain name and finalizer path used to manage garbage collection of the GameServer. + FinalizerName = agones.GroupName + "/controller" + + // NodePodIP identifies an IP address from a pod. + NodePodIP corev1.NodeAddressType = "PodIP" + + // PassthroughPortAssignmentAnnotation is an annotation to keep track of game server container and its Passthrough ports indices + PassthroughPortAssignmentAnnotation = "agones.dev/container-passthrough-port-assignment" + + // True is the string "true" to appease the goconst lint. + True = "true" + // False is the string "false" to appease the goconst lint. + False = "false" +) + +var ( + // GameServerRolePodSelector is the selector to get all GameServer Pods + GameServerRolePodSelector = labels.SelectorFromSet(labels.Set{RoleLabel: GameServerLabelRole}) + + // TerminalGameServerStates is a set (map[GameServerState]bool) of states from which a GameServer will not recover. + // From state diagram at https://agones.dev/site/docs/reference/gameserver/ + TerminalGameServerStates = map[GameServerState]bool{ + GameServerStateShutdown: true, + GameServerStateError: true, + GameServerStateUnhealthy: true, + } +) + +// +genclient +// +genclient:noStatus +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// GameServer is the data structure for a GameServer resource. +// It is worth noting that while there is a `GameServerStatus` Status entry for the `GameServer`, it is not +// defined as a subresource - unlike `Fleet` and other Agones resources. +// This is so that we can retain the ability to change multiple aspects of a `GameServer` in a single atomic operation, +// which is particularly useful for operations such as allocation. +type GameServer struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec GameServerSpec `json:"spec"` + Status GameServerStatus `json:"status"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// GameServerList is a list of GameServer resources +type GameServerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []GameServer `json:"items"` +} + +// GameServerTemplateSpec is a template for GameServers +type GameServerTemplateSpec struct { + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec GameServerSpec `json:"spec"` +} + +// GameServerSpec is the spec for a GameServer resource +type GameServerSpec struct { + // Container specifies which Pod container is the game server. Only required if there is more than one + // container defined + Container string `json:"container,omitempty"` + // Ports are the array of ports that can be exposed via the game server + Ports []GameServerPort `json:"ports,omitempty"` + // Health configures health checking + Health Health `json:"health,omitempty"` + // Scheduling strategy. Defaults to "Packed" + Scheduling apis.SchedulingStrategy `json:"scheduling,omitempty"` + // SdkServer specifies parameters for the Agones SDK Server sidecar container + SdkServer SdkServer `json:"sdkServer,omitempty"` + // Template describes the Pod that will be created for the GameServer + Template corev1.PodTemplateSpec `json:"template"` + // (Alpha, PlayerTracking feature flag) Players provides the configuration for player tracking features. + // +optional + Players *PlayersSpec `json:"players,omitempty"` + // (Beta, CountsAndLists feature flag) Counters provides the configuration for tracking of int64 values against a GameServer. + // Keys must be declared at GameServer creation time. + // +optional + Counters map[string]CounterStatus `json:"counters,omitempty"` + // (Beta, CountsAndLists feature flag) Lists provides the configuration for tracking of lists of up to 1000 values against a GameServer. + // Keys must be declared at GameServer creation time. + // +optional + Lists map[string]ListStatus `json:"lists,omitempty"` + // Eviction specifies the eviction tolerance of the GameServer. Defaults to "Never". + // +optional + Eviction *Eviction `json:"eviction,omitempty"` + // immutableReplicas is present in gameservers.agones.dev but omitted here (it's always 1). +} + +// PlayersSpec tracks the initial player capacity +type PlayersSpec struct { + InitialCapacity int64 `json:"initialCapacity,omitempty"` +} + +// Eviction specifies the eviction tolerance of the GameServer +type Eviction struct { + // Game server supports termination via SIGTERM: + // - Always: Allow eviction for both Cluster Autoscaler and node drain for upgrades + // - OnUpgrade: Allow eviction for upgrades alone + // - Never (default): Pod should run to completion + Safe EvictionSafe `json:"safe,omitempty"` +} + +// Health configures health checking on the GameServer +type Health struct { + // Disabled is whether health checking is disabled or not + Disabled bool `json:"disabled,omitempty"` + // PeriodSeconds is the number of seconds each health ping has to occur in + PeriodSeconds int32 `json:"periodSeconds,omitempty"` + // FailureThreshold how many failures in a row constitutes unhealthy + FailureThreshold int32 `json:"failureThreshold,omitempty"` + // InitialDelaySeconds initial delay before checking health + InitialDelaySeconds int32 `json:"initialDelaySeconds,omitempty"` +} + +// GameServerPort defines a set of Ports that +// are to be exposed via the GameServer +type GameServerPort struct { + // Name is the descriptive name of the port + Name string `json:"name,omitempty"` + // (Alpha, PortRanges feature flag) Range is the port range name from which to select a port when using a + // 'Dynamic' or 'Passthrough' port policy. + // +optional + Range string `json:"range,omitempty"` + // PortPolicy defines the policy for how the HostPort is populated. + // Dynamic port will allocate a HostPort within the selected MIN_PORT and MAX_PORT range passed to the controller + // at installation time. + // When `Static` portPolicy is specified, `HostPort` is required, to specify the port that game clients will + // connect to + // `Passthrough` dynamically sets the `containerPort` to the same value as the dynamically selected hostPort. + // `None` portPolicy ignores `HostPort` and the `containerPort` (optional) is used to set the port on the GameServer instance. + PortPolicy PortPolicy `json:"portPolicy,omitempty"` + // Container is the name of the container or sidecar container on which to open the port. Defaults to the game server container. + // +optional + Container *string `json:"container,omitempty"` + // ContainerPort is the port that is being opened on the specified container's process + ContainerPort int32 `json:"containerPort,omitempty"` + // HostPort the port exposed on the host for clients to connect to + HostPort int32 `json:"hostPort,omitempty"` + // Protocol is the network protocol being used. Defaults to UDP. TCP and TCPUDP are other options. + Protocol corev1.Protocol `json:"protocol,omitempty"` +} + +// SdkServer specifies parameters for the Agones SDK Server sidecar container +type SdkServer struct { + // LogLevel for SDK server (sidecar) logs. Defaults to "Info" + LogLevel SdkServerLogLevel `json:"logLevel,omitempty"` + // GRPCPort is the port on which the SDK Server binds the gRPC server to accept incoming connections + GRPCPort int32 `json:"grpcPort,omitempty"` + // HTTPPort is the port on which the SDK Server binds the HTTP gRPC gateway server to accept incoming connections + HTTPPort int32 `json:"httpPort,omitempty"` +} + +// GameServerStatus is the status for a GameServer resource +type GameServerStatus struct { + // GameServerState is the current state of a GameServer, e.g. Creating, Starting, Ready, etc + State GameServerState `json:"state"` + Ports []GameServerStatusPort `json:"ports"` + Address string `json:"address"` + // Addresses is the array of addresses at which the GameServer can be reached; copy of Node.Status.addresses. + // +optional + Addresses []corev1.NodeAddress `json:"addresses"` + NodeName string `json:"nodeName"` + ReservedUntil *metav1.Time `json:"reservedUntil"` + // [Stage:Alpha] + // [FeatureFlag:PlayerTracking] + // +optional + Players *PlayerStatus `json:"players"` + // (Beta, CountsAndLists feature flag) Counters and Lists provides the configuration for generic tracking features. + // +optional + Counters map[string]CounterStatus `json:"counters,omitempty"` + // +optional + Lists map[string]ListStatus `json:"lists,omitempty"` + // Eviction specifies the eviction tolerance of the GameServer. + // +optional + Eviction *Eviction `json:"eviction,omitempty"` + // immutableReplicas is present in gameservers.agones.dev but omitted here (it's always 1). +} + +// GameServerStatusPort shows the port that was allocated to a +// GameServer. +type GameServerStatusPort struct { + Name string `json:"name,omitempty"` + Port int32 `json:"port"` +} + +// PlayerStatus stores the current player capacity values +type PlayerStatus struct { + Count int64 `json:"count"` + Capacity int64 `json:"capacity"` + IDs []string `json:"ids"` +} + +// CounterStatus stores the current counter values and maximum capacity +type CounterStatus struct { + Count int64 `json:"count"` + Capacity int64 `json:"capacity"` +} + +// ListStatus stores the current list values and maximum capacity +type ListStatus struct { + Capacity int64 `json:"capacity"` + Values []string `json:"values"` +} + +// ApplyDefaults applies default values to the GameServer if they are not already populated +func (gs *GameServer) ApplyDefaults() { + // VersionAnnotation is the annotation that stores + // the version of sdk which runs in a sidecar + if gs.ObjectMeta.Annotations == nil { + gs.ObjectMeta.Annotations = map[string]string{} + } + gs.ObjectMeta.Annotations[VersionAnnotation] = pkg.Version + gs.ObjectMeta.Finalizers = append(gs.ObjectMeta.Finalizers, FinalizerName) + + gs.Spec.ApplyDefaults() + gs.applyStatusDefaults() +} + +// ApplyDefaults applies default values to the GameServerSpec if they are not already populated +func (gss *GameServerSpec) ApplyDefaults() { + gss.applyContainerDefaults() + gss.applyPortDefaults() + gss.applyHealthDefaults() + gss.applyEvictionDefaults() + gss.applySchedulingDefaults() + gss.applySdkServerDefaults() +} + +// applySdkServerDefaults applies the default log level ("Info") for the sidecar +func (gss *GameServerSpec) applySdkServerDefaults() { + if gss.SdkServer.LogLevel == "" { + gss.SdkServer.LogLevel = SdkServerLogLevelInfo + } + if gss.SdkServer.GRPCPort == 0 { + gss.SdkServer.GRPCPort = 9357 + } + if gss.SdkServer.HTTPPort == 0 { + gss.SdkServer.HTTPPort = 9358 + } +} + +// applyContainerDefaults applies the container defaults +func (gss *GameServerSpec) applyContainerDefaults() { + if len(gss.Template.Spec.Containers) == 1 { + gss.Container = gss.Template.Spec.Containers[0].Name + } +} + +// applyHealthDefaults applies health checking defaults +func (gss *GameServerSpec) applyHealthDefaults() { + if !gss.Health.Disabled { + if gss.Health.PeriodSeconds <= 0 { + gss.Health.PeriodSeconds = 5 + } + if gss.Health.FailureThreshold <= 0 { + gss.Health.FailureThreshold = 3 + } + if gss.Health.InitialDelaySeconds <= 0 { + gss.Health.InitialDelaySeconds = 5 + } + } +} + +// applyStatusDefaults applies Status defaults +func (gs *GameServer) applyStatusDefaults() { + if gs.Status.State == "" { + gs.Status.State = GameServerStateCreating + // applyStatusDefaults() should be called after applyPortDefaults() + if gs.HasPortPolicy(Dynamic) || gs.HasPortPolicy(Passthrough) { + gs.Status.State = GameServerStatePortAllocation + } + } + + if runtime.FeatureEnabled(runtime.FeaturePlayerTracking) { + // set value if enabled, otherwise very easy to accidentally panic + // when gs.Status.Players is nil + if gs.Status.Players == nil { + gs.Status.Players = &PlayerStatus{} + } + if gs.Spec.Players != nil { + gs.Status.Players.Capacity = gs.Spec.Players.InitialCapacity + } + } + + gs.applyEvictionStatus() + gs.applyCountsListsStatus() +} + +// applyPortDefaults applies default values for all ports +func (gss *GameServerSpec) applyPortDefaults() { + for i, p := range gss.Ports { + // basic spec + if p.PortPolicy == "" { + gss.Ports[i].PortPolicy = Dynamic + } + + if p.Range == "" { + gss.Ports[i].Range = DefaultPortRange + } + + if p.Protocol == "" { + gss.Ports[i].Protocol = "UDP" + } + + if p.Container == nil || *p.Container == "" { + gss.Ports[i].Container = &gss.Container + } + } +} + +func (gss *GameServerSpec) applySchedulingDefaults() { + if gss.Scheduling == "" { + gss.Scheduling = apis.Packed + } +} + +func (gss *GameServerSpec) applyEvictionDefaults() { + if gss.Eviction == nil { + gss.Eviction = &Eviction{} + } + if gss.Eviction.Safe == "" { + gss.Eviction.Safe = EvictionSafeNever + } +} + +func (gs *GameServer) applyEvictionStatus() { + gs.Status.Eviction = gs.Spec.Eviction.DeepCopy() + if gs.Spec.Template.ObjectMeta.Annotations[PodSafeToEvictAnnotation] == "true" { + if gs.Status.Eviction == nil { + gs.Status.Eviction = &Eviction{} + } + gs.Status.Eviction.Safe = EvictionSafeAlways + } +} + +func (gs *GameServer) applyCountsListsStatus() { + if !runtime.FeatureEnabled(runtime.FeatureCountsAndLists) { + return + } + if gs.Spec.Counters != nil { + countersCopy := make(map[string]CounterStatus, len(gs.Spec.Counters)) + for key, val := range gs.Spec.Counters { + countersCopy[key] = *val.DeepCopy() + } + gs.Status.Counters = countersCopy + } + if gs.Spec.Lists != nil { + listsCopy := make(map[string]ListStatus, len(gs.Spec.Lists)) + for key, val := range gs.Spec.Lists { + listsCopy[key] = *val.DeepCopy() + } + gs.Status.Lists = listsCopy + } +} + +// validateFeatureGates checks if fields are set when the associated feature gate is not set. +func (gss *GameServerSpec) validateFeatureGates(fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + if !runtime.FeatureEnabled(runtime.FeaturePlayerTracking) { + if gss.Players != nil { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("players"), fmt.Sprintf("Value cannot be set unless feature flag %s is enabled", runtime.FeaturePlayerTracking))) + } + } + + if !runtime.FeatureEnabled(runtime.FeatureCountsAndLists) { + if gss.Counters != nil { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("counters"), fmt.Sprintf("Value cannot be set unless feature flag %s is enabled", runtime.FeatureCountsAndLists))) + } + if gss.Lists != nil { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("lists"), fmt.Sprintf("Value cannot be set unless feature flag %s is enabled", runtime.FeatureCountsAndLists))) + } + } + + + return allErrs +} + +// Validate validates the GameServerSpec configuration. +// devAddress is a specific IP address used for local Gameservers, for fleets "" is used +// If a GameServer Spec is invalid there will be > 0 values in the returned array +func (gss *GameServerSpec) Validate(apiHooks APIHooks, devAddress string, fldPath *field.Path) field.ErrorList { + allErrs := gss.validateFeatureGates(fldPath) + if len(devAddress) > 0 { + // verify that the value is a valid IP address. + if net.ParseIP(devAddress) == nil { + // Authentication is only required if the gameserver is created directly. + allErrs = append(allErrs, field.Invalid(field.NewPath("metadata", "annotations", DevAddressAnnotation), devAddress, "must be a valid IP address")) + } + + for i, p := range gss.Ports { + if p.HostPort == 0 { + allErrs = append(allErrs, field.Required(fldPath.Child("ports").Index(i).Child("hostPort"), DevAddressAnnotation)) + } + if p.PortPolicy != Static { + allErrs = append(allErrs, field.Required(fldPath.Child("ports").Index(i).Child("portPolicy"), ErrPortPolicyStatic)) + } + } + + allErrs = append(allErrs, validateObjectMeta(&gss.Template.ObjectMeta, fldPath.Child("template", "metadata"))...) + return allErrs + } + + // make sure a name is specified when there is multiple containers in the pod. + if gss.Container == "" && len(gss.Template.Spec.Containers) > 1 { + allErrs = append(allErrs, field.Required(fldPath.Child("container"), ErrContainerRequired)) + } + + // make sure the container value points to a valid container + if !gss.HasContainer(gss.Container, false) { + allErrs = append(allErrs, + field.Invalid( + fldPath.Child("container"), + gss.Container, + "Could not find a container named "+gss.Container, + ), + ) + } + + // no host port when using dynamic PortPolicy + for i, p := range gss.Ports { + path := fldPath.Child("ports").Index(i) + if p.PortPolicy == Dynamic || p.PortPolicy == Static { + if p.ContainerPort <= 0 { + allErrs = append(allErrs, field.Required(path.Child("containerPort"), ErrContainerPortRequired)) + } + } + + if p.PortPolicy == Passthrough && p.ContainerPort > 0 { + allErrs = append(allErrs, field.Required(path.Child("containerPort"), ErrContainerPortPassthrough)) + } + + if p.HostPort > 0 && (p.PortPolicy == Dynamic || p.PortPolicy == Passthrough) { + allErrs = append(allErrs, field.Forbidden(path.Child("hostPort"), ErrHostPort)) + } + + if p.Container != nil && gss.Container != "" { + if !gss.HasContainer(*p.Container, true) { + allErrs = append(allErrs, field.Invalid(path.Child("container"), *p.Container, ErrContainerNameInvalid)) + } + } + } + for i, c := range gss.Template.Spec.InitContainers { + path := fldPath.Child("template", "spec", "initContainers").Index(i) + allErrs = append(allErrs, ValidateResourceRequirements(&c.Resources, path.Child("resources"))...) + } + for i, c := range gss.Template.Spec.Containers { + path := fldPath.Child("template", "spec", "containers").Index(i) + allErrs = append(allErrs, ValidateResourceRequirements(&c.Resources, path.Child("resources"))...) + } + + allErrs = append(allErrs, apiHooks.ValidateGameServerSpec(gss, fldPath)...) + allErrs = append(allErrs, validateObjectMeta(&gss.Template.ObjectMeta, fldPath.Child("template", "metadata"))...) + return allErrs +} + +// ValidateResourceRequirements Validates resource requirement spec. +func ValidateResourceRequirements(requirements *corev1.ResourceRequirements, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + limPath := fldPath.Child("limits") + reqPath := fldPath.Child("requests") + + for resourceName, quantity := range requirements.Limits { + fldPath := limPath.Key(string(resourceName)) + // Validate resource quantity. + allErrs = append(allErrs, ValidateNonnegativeQuantity(quantity, fldPath)...) + + } + + for resourceName, quantity := range requirements.Requests { + fldPath := reqPath.Key(string(resourceName)) + // Validate resource quantity. + allErrs = append(allErrs, ValidateNonnegativeQuantity(quantity, fldPath)...) + + // Check that request <= limit. + limitQuantity, exists := requirements.Limits[resourceName] + if exists && quantity.Cmp(limitQuantity) > 0 { + allErrs = append(allErrs, field.Invalid(reqPath, quantity.String(), fmt.Sprintf("must be less than or equal to %s limit of %s", resourceName, limitQuantity.String()))) + } + } + return allErrs +} + +// ValidateNonnegativeQuantity Validates that a Quantity is not negative +func ValidateNonnegativeQuantity(value resource.Quantity, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + if value.Cmp(resource.Quantity{}) < 0 { + allErrs = append(allErrs, field.Invalid(fldPath, value.String(), apimachineryvalidation.IsNegativeErrorMsg)) + } + return allErrs +} + +// Validate validates the GameServer configuration. +// If a GameServer is invalid there will be > 0 values in +// the returned array +func (gs *GameServer) Validate(apiHooks APIHooks) field.ErrorList { + allErrs := validateName(gs, field.NewPath("metadata")) + + // make sure the host port is specified if this is a development server + devAddress, _ := gs.GetDevAddress() + allErrs = append(allErrs, gs.Spec.Validate(apiHooks, devAddress, field.NewPath("spec"))...) + return allErrs +} + +// GetDevAddress returns the address for game server. +func (gs *GameServer) GetDevAddress() (string, bool) { + devAddress, hasDevAddress := gs.ObjectMeta.Annotations[DevAddressAnnotation] + return devAddress, hasDevAddress +} + +// IsDeletable returns false if the server is currently allocated/reserved and is not already in the +// process of being deleted +func (gs *GameServer) IsDeletable() bool { + if gs.Status.State == GameServerStateAllocated || gs.Status.State == GameServerStateReserved { + return !gs.ObjectMeta.DeletionTimestamp.IsZero() + } + + return true +} + +// IsBeingDeleted returns true if the server is in the process of being deleted. +func (gs *GameServer) IsBeingDeleted() bool { + return !gs.ObjectMeta.DeletionTimestamp.IsZero() || gs.Status.State == GameServerStateShutdown +} + +// IsBeforeReady returns true if the GameServer Status has yet to move to or past the Ready +// state in its lifecycle, such as Allocated or Reserved, or any of the Error/Unhealthy states +func (gs *GameServer) IsBeforeReady() bool { + switch gs.Status.State { + case GameServerStatePortAllocation: + return true + case GameServerStateCreating: + return true + case GameServerStateStarting: + return true + case GameServerStateScheduled: + return true + case GameServerStateRequestReady: + return true + } + + return false +} + +// IsActive returns true if the GameServer status is Ready, Reserved, or Allocated state. +func (gs *GameServer) IsActive() bool { + switch gs.Status.State { + case GameServerStateAllocated: + return true + case GameServerStateReady: + return true + case GameServerStateReserved: + return true + } + + return false +} + +// HasContainer determines if the GameServerSpec has a container with the specified name. +// Init containers with RestartPolicy `Always` will be considered if `includeSidecar` is true. +func (gss *GameServerSpec) HasContainer(name string, includeSidecar bool) bool { + for _, c := range gss.Template.Spec.Containers { + if c.Name == name { + return true + } + } + if !includeSidecar { + return false + } + + for _, c := range gss.Template.Spec.InitContainers { + if c.RestartPolicy == nil || *c.RestartPolicy != corev1.ContainerRestartPolicyAlways { + continue + } + if c.Name == name { + return true + } + } + return false +} + +// ApplyToPodContainer applies func(v1.Container) to the specified container in the pod. +// Init containers with RestartPolicy Always will also be considered. +// Returns an error if the container is not found. +func (gs *GameServer) ApplyToPodContainer(pod *corev1.Pod, containerName string, f func(corev1.Container) corev1.Container) error { + for i, c := range pod.Spec.Containers { + if c.Name == containerName { + pod.Spec.Containers[i] = f(c) + return nil + } + } + for i, c := range pod.Spec.InitContainers { + if c.RestartPolicy == nil || *c.RestartPolicy != corev1.ContainerRestartPolicyAlways { + continue + } + if c.Name == containerName { + pod.Spec.InitContainers[i] = f(c) + return nil + } + } + return errors.Errorf("failed to find container named %s in pod spec", containerName) +} + +// Pod creates a new Pod from the PodTemplateSpec +// attached to the GameServer resource +func (gs *GameServer) Pod(apiHooks APIHooks, sidecars ...corev1.Container) (*corev1.Pod, error) { + pod := &corev1.Pod{ + ObjectMeta: *gs.Spec.Template.ObjectMeta.DeepCopy(), + Spec: *gs.Spec.Template.Spec.DeepCopy(), + } + + if len(pod.Spec.Hostname) == 0 { + // replace . with - since it must match RFC 1123 + pod.Spec.Hostname = strings.ReplaceAll(gs.ObjectMeta.Name, ".", "-") + } + + gs.podObjectMeta(pod) + + passthroughContainerPortMap := make(map[string][]int) + for _, p := range gs.Spec.Ports { + var hostPort int32 + portIdx := 0 + + if !runtime.FeatureEnabled(runtime.FeaturePortPolicyNone) || p.PortPolicy != None { + hostPort = p.HostPort + } + + cp := corev1.ContainerPort{ + ContainerPort: p.ContainerPort, + HostPort: hostPort, + Protocol: p.Protocol, + } + err := gs.ApplyToPodContainer(pod, *p.Container, func(c corev1.Container) corev1.Container { + portIdx = len(c.Ports) + c.Ports = append(c.Ports, cp) + + return c + }) + if err != nil { + return nil, err + } + if p.PortPolicy == Passthrough { + passthroughContainerPortMap[*p.Container] = append(passthroughContainerPortMap[*p.Container], portIdx) + } + } + + if len(passthroughContainerPortMap) != 0 { + containerToPassthroughMapJSON, err := json.Marshal(passthroughContainerPortMap) + if err != nil { + return nil, err + } + pod.ObjectMeta.Annotations[PassthroughPortAssignmentAnnotation] = string(containerToPassthroughMapJSON) + } + + if runtime.FeatureEnabled(runtime.FeatureSidecarContainers) { + // make sure all sidecars have a restart policy of Always, so they are valid sidecar containers. + // https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/#sidecar-containers-and-pod-lifecycle + always := corev1.ContainerRestartPolicyAlways + for i := range sidecars { + sidecars[i].RestartPolicy = &always + } + + // addSidecarsAsInitContainers puts the sidecars in the initContainers list so that they can have their own independent + // container restart policies. + pod.Spec.InitContainers = slices.Concat(sidecars, pod.Spec.InitContainers) + + // default GameServer container should also be Restart: Never + if len(pod.Spec.RestartPolicy) == 0 { + pod.Spec.RestartPolicy = corev1.RestartPolicyNever + } + } else { + gs.addSidecarsAsContainers(sidecars, pod) + } + + gs.podScheduling(pod) + + if err := apiHooks.MutateGameServerPod(&gs.Spec, pod); err != nil { + return nil, err + } + if err := apiHooks.SetEviction(gs.Status.Eviction, pod); err != nil { + return nil, err + } + + return pod, nil +} + +// addSidecarsAsContainers puts the sidecars at the start of the general list of containers so that the kubelet starts +// them first +func (gs *GameServer) addSidecarsAsContainers(sidecars []corev1.Container, pod *corev1.Pod) { + containers := make([]corev1.Container, 0, len(sidecars)+len(pod.Spec.Containers)) + containers = append(containers, sidecars...) + containers = append(containers, pod.Spec.Containers...) + pod.Spec.Containers = containers +} + +// podObjectMeta configures the pod ObjectMeta details +func (gs *GameServer) podObjectMeta(pod *corev1.Pod) { + pod.ObjectMeta.GenerateName = "" + // Pods inherit the name of their gameserver. It's safe since there's + // a guarantee that pod won't outlive its parent. + pod.ObjectMeta.Name = gs.ObjectMeta.Name + // Pods for GameServers need to stay in the same namespace + pod.ObjectMeta.Namespace = gs.ObjectMeta.Namespace + // Make sure these are blank, just in case + pod.ObjectMeta.ResourceVersion = "" + pod.ObjectMeta.UID = "" + if pod.ObjectMeta.Labels == nil { + pod.ObjectMeta.Labels = make(map[string]string, 2) + } + if pod.ObjectMeta.Annotations == nil { + pod.ObjectMeta.Annotations = make(map[string]string, 2) + } + pod.ObjectMeta.Labels[RoleLabel] = GameServerLabelRole + // store the GameServer name as a label, for easy lookup later on + pod.ObjectMeta.Labels[GameServerPodLabel] = gs.ObjectMeta.Name + // store the GameServer container as an annotation, to make lookup at a Pod level easier + pod.ObjectMeta.Annotations[GameServerContainerAnnotation] = gs.Spec.Container + ref := metav1.NewControllerRef(gs, SchemeGroupVersion.WithKind("GameServer")) + pod.ObjectMeta.OwnerReferences = append(pod.ObjectMeta.OwnerReferences, *ref) + + // Add Agones version into Pod Annotations + pod.ObjectMeta.Annotations[VersionAnnotation] = pkg.Version +} + +// podScheduling applies the Fleet scheduling strategy to the passed in Pod +// this sets the a PreferredDuringSchedulingIgnoredDuringExecution for GameServer +// pods to a host topology. Basically doing a half decent job of packing GameServer +// pods together. +func (gs *GameServer) podScheduling(pod *corev1.Pod) { + if gs.Spec.Scheduling == apis.Packed { + if pod.Spec.Affinity == nil { + pod.Spec.Affinity = &corev1.Affinity{} + } + if pod.Spec.Affinity.PodAffinity == nil { + pod.Spec.Affinity.PodAffinity = &corev1.PodAffinity{} + } + + wpat := corev1.WeightedPodAffinityTerm{ + Weight: 100, + PodAffinityTerm: corev1.PodAffinityTerm{ + TopologyKey: "kubernetes.io/hostname", + LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{RoleLabel: GameServerLabelRole}}, + }, + } + + pod.Spec.Affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append(pod.Spec.Affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution, wpat) + } +} + +// DisableServiceAccount disables the service account for the gameserver container +func (gs *GameServer) DisableServiceAccount(pod *corev1.Pod) error { + // gameservers don't get access to the k8s api. + emptyVol := corev1.Volume{Name: "empty", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}} + pod.Spec.Volumes = append(pod.Spec.Volumes, emptyVol) + mount := corev1.VolumeMount{MountPath: "/var/run/secrets/kubernetes.io/serviceaccount", Name: emptyVol.Name, ReadOnly: true} + + return gs.ApplyToPodContainer(pod, gs.Spec.Container, func(c corev1.Container) corev1.Container { + c.VolumeMounts = append(c.VolumeMounts, mount) + + return c + }) +} + +// HasPortPolicy checks if there is a port with a given +// PortPolicy +func (gs *GameServer) HasPortPolicy(policy PortPolicy) bool { + for _, p := range gs.Spec.Ports { + if p.PortPolicy == policy { + return true + } + } + return false +} + +// Status returns a GameServerStatusPort for this GameServerPort +func (p GameServerPort) Status() GameServerStatusPort { + if p.PortPolicy == None { + return GameServerStatusPort{Name: p.Name, Port: p.ContainerPort} + } + + return GameServerStatusPort{Name: p.Name, Port: p.HostPort} +} + +// CountPorts returns the number of +// ports that match condition function +func (gs *GameServer) CountPorts(f func(policy PortPolicy) bool) int { + count := 0 + for _, p := range gs.Spec.Ports { + if f(p.PortPolicy) { + count++ + } + } + return count +} + +// CountPortsForRange returns the number of ports that match condition function and range name. +func (gs *GameServer) CountPortsForRange(name string, f func(policy PortPolicy) bool) int { + count := 0 + for _, p := range gs.Spec.Ports { + if p.Range == name && f(p.PortPolicy) { + count++ + } + } + return count +} + +// Patch creates a JSONPatch to move the current GameServer to the passed in delta GameServer. +// Returned Patch includes a "test" operation that will cause the GameServers.Patch() operation to +// fail if the Game Server has been updated (ResourceVersion has changed) in between when the Patch +// was created and applied. +func (gs *GameServer) Patch(delta *GameServer) ([]byte, error) { + var result []byte + + oldJSON, err := json.Marshal(gs) + if err != nil { + return result, errors.Wrapf(err, "error marshalling to json current GameServer %s", gs.ObjectMeta.Name) + } + + newJSON, err := json.Marshal(delta) + if err != nil { + return result, errors.Wrapf(err, "error marshalling to json delta GameServer %s", delta.ObjectMeta.Name) + } + + patch, err := jsonpatch.CreatePatch(oldJSON, newJSON) + if err != nil { + return result, errors.Wrapf(err, "error creating patch for GameServer %s", gs.ObjectMeta.Name) + } + + // Per https://jsonpatch.com/ "Tests that the specified value is set in the document. If the test + // fails, then the patch as a whole should not apply." + // Used here to check the object has not been updated (has not changed ResourceVersion). + patches := []jsonpatch.JsonPatchOperation{{Operation: "test", Path: "/metadata/resourceVersion", Value: gs.ObjectMeta.ResourceVersion}} + patches = append(patches, patch...) + + result, err = json.Marshal(patches) + return result, errors.Wrapf(err, "error creating json for patch for GameServer %s", gs.ObjectMeta.Name) +} + +// UpdateCount increments or decrements a CounterStatus on a Game Server by the given amount. +func (gs *GameServer) UpdateCount(name string, action string, amount int64) error { + if !(action == GameServerPriorityIncrement || action == GameServerPriorityDecrement) { + return errors.Errorf("unable to UpdateCount with Name %s, Action %s, Amount %d. Allocation action must be one of %s or %s", name, action, amount, GameServerPriorityIncrement, GameServerPriorityDecrement) + } + if amount < 0 { + return errors.Errorf("unable to UpdateCount with Name %s, Action %s, Amount %d. Amount must be greater than 0", name, action, amount) + } + if counter, ok := gs.Status.Counters[name]; ok { + cnt := counter.Count + if action == GameServerPriorityIncrement { + cnt += amount + } else { + cnt -= amount + } + // Truncate to Capacity if Count > Capacity + if cnt > counter.Capacity { + cnt = counter.Capacity + } + // Truncate to Zero if Count is negative + if cnt < 0 { + cnt = 0 + } + counter.Count = cnt + gs.Status.Counters[name] = counter + return nil + } + return errors.Errorf("unable to UpdateCount with Name %s, Action %s, Amount %d. Counter not found in GameServer %s", name, action, amount, gs.ObjectMeta.GetName()) +} + +// UpdateCounterCapacity updates the CounterStatus Capacity to the given capacity. +func (gs *GameServer) UpdateCounterCapacity(name string, capacity int64) error { + if capacity < 0 { + return errors.Errorf("unable to UpdateCounterCapacity: Name %s, Capacity %d. Capacity must be greater than or equal to 0", name, capacity) + } + if counter, ok := gs.Status.Counters[name]; ok { + counter.Capacity = capacity + // If Capacity is now less than Count, reset Count here to equal Capacity + if counter.Count > counter.Capacity { + counter.Count = counter.Capacity + } + gs.Status.Counters[name] = counter + return nil + } + return errors.Errorf("unable to UpdateCounterCapacity: Name %s, Capacity %d. Counter not found in GameServer %s", name, capacity, gs.ObjectMeta.GetName()) +} + +// UpdateListCapacity updates the ListStatus Capacity to the given capacity. +func (gs *GameServer) UpdateListCapacity(name string, capacity int64) error { + + if capacity < 0 || capacity > ListMaxCapacity { + return errors.Errorf("unable to UpdateListCapacity: Name %s, Capacity %d. Capacity must be between 0 and 1000, inclusive", name, capacity) + } + if list, ok := gs.Status.Lists[name]; ok { + list.Capacity = capacity + list.Values = truncateList(list.Capacity, list.Values) + gs.Status.Lists[name] = list + return nil + } + return errors.Errorf("unable to UpdateListCapacity: Name %s, Capacity %d. List not found in GameServer %s", name, capacity, gs.ObjectMeta.GetName()) +} + +// AppendListValues adds unique values to the ListStatus Values list. +func (gs *GameServer) AppendListValues(name string, values []string) error { + if values == nil { + return errors.Errorf("unable to AppendListValues: Name %s, Values %s. Values must not be nil", name, values) + } + if list, ok := gs.Status.Lists[name]; ok { + mergedList := MergeRemoveDuplicates(list.Values, values) + // Any duplicate values are silently dropped. + list.Values = mergedList + list.Values = truncateList(list.Capacity, list.Values) + gs.Status.Lists[name] = list + return nil + } + return errors.Errorf("unable to AppendListValues: Name %s, Values %s. List not found in GameServer %s", name, values, gs.ObjectMeta.GetName()) +} + +// DeleteListValues removes values from the ListStatus Values list. Values in the DeleteListValues +// list that are not in the ListStatus Values list are ignored. +func (gs *GameServer) DeleteListValues(name string, values []string) error { + if values == nil { + return errors.Errorf("unable to DeleteListValues: Name %s, Values %s. Values must not be nil", name, values) + } + if list, ok := gs.Status.Lists[name]; ok { + deleteValuesMap := make(map[string]bool) + for _, value := range values { + deleteValuesMap[value] = true + } + newList := deleteValues(list.Values, deleteValuesMap) + list.Values = newList + gs.Status.Lists[name] = list + return nil + } + return errors.Errorf("unable to DeleteListValues: Name %s, Values %s. List not found in GameServer %s", name, values, gs.ObjectMeta.GetName()) +} + +// deleteValues returns a new list with all the values in valuesList that are not keys in deleteValuesMap. +func deleteValues(valuesList []string, deleteValuesMap map[string]bool) []string { + newValuesList := []string{} + for _, value := range valuesList { + if _, ok := deleteValuesMap[value]; ok { + continue + } + newValuesList = append(newValuesList, value) + } + return newValuesList +} + +// truncateList truncates the list to the given capacity +func truncateList(capacity int64, list []string) []string { + if list == nil || len(list) <= int(capacity) { + return list + } + list = append([]string{}, list[:capacity]...) + return list +} + +// MergeRemoveDuplicates merges two lists and removes any duplicate values. +// Maintains ordering, so new values from list2 are appended to the end of list1. +// Returns a new list with unique values only. +func MergeRemoveDuplicates(list1 []string, list2 []string) []string { + uniqueList := []string{} + listMap := make(map[string]bool) + for _, v1 := range list1 { + if _, ok := listMap[v1]; !ok { + uniqueList = append(uniqueList, v1) + listMap[v1] = true + } + } + for _, v2 := range list2 { + if _, ok := listMap[v2]; !ok { + uniqueList = append(uniqueList, v2) + listMap[v2] = true + } + } + return uniqueList +} + +// CompareCountAndListPriorities compares two game servers based on a list of CountsAndLists Priorities using available +// capacity as the comparison. +func (gs *GameServer) CompareCountAndListPriorities(priorities []Priority, other *GameServer) *bool { + for _, priority := range priorities { + res := gs.compareCountAndListPriority(&priority, other) + if res != nil { + // reverse if descending + if priority.Order == GameServerPriorityDescending { + flip := !*res + return &flip + } + + return res + } + } + + return nil +} + +// compareCountAndListPriority compares two game servers based on a CountsAndLists Priority using available +// capacity (Capacity - Count for Counters, and Capacity - len(Values) for Lists) as the comparison. +// Returns true if gs1 < gs2; false if gs1 > gs2; nil if gs1 == gs2; nil if neither gamer server has the Priority. +// If only one game server has the Priority, prefer that server. I.e. nil < gsX when Priority +// Order is Descending (3, 2, 1, 0, nil), and nil > gsX when Order is Ascending (0, 1, 2, 3, nil). +func (gs *GameServer) compareCountAndListPriority(p *Priority, other *GameServer) *bool { + var gs1ok, gs2ok bool + t := true + f := false + switch p.Type { + case GameServerPriorityCounter: + // Check if both game servers contain the Counter. + counter1, ok1 := gs.Status.Counters[p.Key] + counter2, ok2 := other.Status.Counters[p.Key] + // If both game servers have the Counter + if ok1 && ok2 { + availCapacity1 := counter1.Capacity - counter1.Count + availCapacity2 := counter2.Capacity - counter2.Count + if availCapacity1 < availCapacity2 { + return &t + } + if availCapacity1 > availCapacity2 { + return &f + } + if availCapacity1 == availCapacity2 { + return nil + } + } + gs1ok = ok1 + gs2ok = ok2 + case GameServerPriorityList: + // Check if both game servers contain the List. + list1, ok1 := gs.Status.Lists[p.Key] + list2, ok2 := other.Status.Lists[p.Key] + // If both game servers have the List + if ok1 && ok2 { + availCapacity1 := list1.Capacity - int64(len(list1.Values)) + availCapacity2 := list2.Capacity - int64(len(list2.Values)) + if availCapacity1 < availCapacity2 { + return &t + } + if availCapacity1 > availCapacity2 { + return &f + } + if availCapacity1 == availCapacity2 { + return nil + } + } + gs1ok = ok1 + gs2ok = ok2 + } + // If only one game server has the Priority, prefer that server. I.e. nil < gsX when Order is + // Descending (3, 2, 1, 0, nil), and nil > gsX when Order is Ascending (0, 1, 2, 3, nil). + if (gs1ok && p.Order == GameServerPriorityDescending) || + (gs2ok && p.Order == GameServerPriorityAscending) { + return &f + } + if (gs1ok && p.Order == GameServerPriorityAscending) || + (gs2ok && p.Order == GameServerPriorityDescending) { + return &t + } + // If neither game server has the Priority + return nil +} diff --git a/pkg/apis/agones/v1/gameserver_test.go b/pkg/apis/agones/v1/gameserver_test.go index 9a05df2ea0..e63df56df5 100644 --- a/pkg/apis/agones/v1/gameserver_test.go +++ b/pkg/apis/agones/v1/gameserver_test.go @@ -71,7 +71,6 @@ func TestStatus(t *testing.T) { } runtime.FeatureTestMutex.Lock() defer runtime.FeatureTestMutex.Unlock() - require.NoError(t, runtime.ParseFeatures(string(runtime.FeaturePortPolicyNone)+"=true")) for _, tc := range testCases { name := "test-name" @@ -1536,35 +1535,8 @@ func TestGameServerValidateFeatures(t *testing.T) { }, }, }, - { - description: "PortPolicyNone is disabled, PortPolicy field set to None", - feature: fmt.Sprintf("%s=false", runtime.FeaturePortPolicyNone), - gs: GameServer{ - Spec: GameServerSpec{ - Ports: []GameServerPort{ - { - Name: "main", - ContainerPort: 7777, - PortPolicy: None, - }, - }, - Container: "testing", - Lists: map[string]ListStatus{}, - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "testing", Image: "testing/image"}}}, - }, - }, - }, - want: field.ErrorList{ - field.Forbidden( - field.NewPath("spec.ports[0].portPolicy"), - "Value cannot be set to None unless feature flag PortPolicyNone is enabled", - ), - }, - }, { description: "PortPolicyNone is enabled, PortPolicy field set to None", - feature: fmt.Sprintf("%s=true", runtime.FeaturePortPolicyNone), gs: GameServer{ Spec: GameServerSpec{ Ports: []GameServerPort{ @@ -1583,12 +1555,8 @@ func TestGameServerValidateFeatures(t *testing.T) { }, }, } - for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { - err := runtime.ParseFeatures(tc.feature) - assert.NoError(t, err) - errs := tc.gs.Validate(fakeAPIHooks{}) assert.ElementsMatch(t, tc.want, errs, "ErrorList check") }) diff --git a/pkg/cloudproduct/gke/gke.go b/pkg/cloudproduct/gke/gke.go index 4b7963d54f..1711d13c7f 100644 --- a/pkg/cloudproduct/gke/gke.go +++ b/pkg/cloudproduct/gke/gke.go @@ -1,324 +1,321 @@ -// Copyright Contributors to Agones a Series of LF Projects, LLC. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package gke implements the GKE cloud product (specifically Autopilot for now) -package gke - -import ( - "context" - "encoding/json" - "fmt" - - "agones.dev/agones/pkg/apis" - agonesv1 "agones.dev/agones/pkg/apis/agones/v1" - "agones.dev/agones/pkg/client/informers/externalversions" - "agones.dev/agones/pkg/cloudproduct/eviction" - "agones.dev/agones/pkg/portallocator" - "agones.dev/agones/pkg/util/runtime" - "cloud.google.com/go/compute/metadata" - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/validation/field" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" -) - -const ( - hostPortAssignmentAnnotation = "autopilot.gke.io/host-port-assignment" - primaryContainerAnnotation = "autopilot.gke.io/primary-container" - - errRangeInvalid = "range must not be used on GKE Autopilot" - errSchedulingMustBePacked = "scheduling strategy must be Packed on GKE Autopilot" - errEvictionSafeOnUpgradeInvalid = "eviction.safe OnUpgrade not supported on GKE Autopilot" -) - -var ( - autopilotMutatingWebhooks = []string{ - "workload-defaulter.config.common-webhooks.networking.gke.io", // pre-1.26 - "sasecret-redacter.config.common-webhooks.networking.gke.io", // 1.26+ - } - noWorkloadDefaulter = fmt.Sprintf("found no MutatingWebhookConfigurations matching %v", autopilotMutatingWebhooks) - - logger = runtime.NewLoggerWithSource("gke") -) - -type gkeAutopilot struct { - useExtendedDurationPods bool -} - -// hostPortAssignment is the JSON structure of the `host-port-assignment` annotation -// -//nolint:govet // API-like, keep consistent -type hostPortAssignment struct { - Min int32 `json:"min,omitempty"` - Max int32 `json:"max,omitempty"` - PortsAssigned map[int32]int32 `json:"portsAssigned,omitempty"` // old -> new -} - -// Detect whether we're running on GKE and/or Autopilot and return the appropriate -// cloud product string. -func Detect(ctx context.Context, kc *kubernetes.Clientset) string { - if !metadata.OnGCE() { - return "" - } - // Look for the workload defaulter - this is the current best method to detect Autopilot - found := false - for _, webhook := range autopilotMutatingWebhooks { - if _, err := kc.AdmissionregistrationV1().MutatingWebhookConfigurations().Get( - ctx, webhook, metav1.GetOptions{}); err != nil { - logger.WithError(err).WithField("webhook", webhook).Info("Detecting Autopilot MutatingWebhookConfiguration") - } else { - found = true - break - } - } - if !found { - logger.WithField("reason", noWorkloadDefaulter).Info( - "Assuming GKE Standard and defaulting to generic provider (expected if not on GKE Autopilot)") - return "" // GKE standard, but we don't need an interface for it just yet. - } - logger.Info("Running on GKE Autopilot (skip detection with --cloud-product=gke-autopilot)") - return "gke-autopilot" -} - -// Autopilot returns a GKE Autopilot cloud product -// -//nolint:revive // ignore the unexported return; implements ControllerHooksInterface -func Autopilot() *gkeAutopilot { - return &gkeAutopilot{useExtendedDurationPods: runtime.FeatureEnabled(runtime.FeatureGKEAutopilotExtendedDurationPods)} -} - -func (*gkeAutopilot) SyncPodPortsToGameServer(gs *agonesv1.GameServer, pod *corev1.Pod) error { - // If applyGameServerAddressAndPort has already filled in Status, SyncPodPortsToGameServer - // has already run. Skip syncing from the Pod again - this avoids having to reason - // about whether we're re-applying the old->new mapping. - if len(gs.Status.Ports) == len(gs.Spec.Ports) { - return nil - } - annotation, ok := pod.ObjectMeta.Annotations[hostPortAssignmentAnnotation] - if !ok { - return nil - } - var hpa hostPortAssignment - if err := json.Unmarshal([]byte(annotation), &hpa); err != nil { - return errors.Wrapf(err, "could not unmarshal annotation %s (value %q)", hostPortAssignmentAnnotation, annotation) - } - for i, p := range gs.Spec.Ports { - if newPort, ok := hpa.PortsAssigned[p.HostPort]; ok { - gs.Spec.Ports[i].HostPort = newPort - } - } - return nil -} - -func (*gkeAutopilot) NewPortAllocator(portRanges map[string]portallocator.PortRange, - _ informers.SharedInformerFactory, - _ externalversions.SharedInformerFactory, -) portallocator.Interface { - defPortRange := portRanges[agonesv1.DefaultPortRange] - return &autopilotPortAllocator{minPort: defPortRange.MinPort, maxPort: defPortRange.MaxPort} -} - -func (*gkeAutopilot) WaitOnFreePorts() bool { return true } - -func (g *gkeAutopilot) ValidateGameServerSpec(gss *agonesv1.GameServerSpec, fldPath *field.Path) field.ErrorList { - allErrs := g.ValidateScheduling(gss.Scheduling, fldPath.Child("scheduling")) - - // Loop through ports and use the helper function for validation - for i, p := range gss.Ports { - allErrs = append(allErrs, validatePortPolicy(p, i, fldPath)...) - - } - - // See SetEviction comment below for why we block EvictionSafeOnUpgrade, if Extended Duration pods aren't supported. - if !g.useExtendedDurationPods && gss.Eviction.Safe == agonesv1.EvictionSafeOnUpgrade { - allErrs = append(allErrs, field.Invalid(fldPath.Child("eviction").Child("safe"), string(gss.Eviction.Safe), errEvictionSafeOnUpgradeInvalid)) - } - return allErrs -} - -func (*gkeAutopilot) ValidateScheduling(ss apis.SchedulingStrategy, fldPath *field.Path) field.ErrorList { - if ss != apis.Packed { - return field.ErrorList{ - field.Invalid(fldPath, string(ss), errSchedulingMustBePacked), - } - } - return nil -} - -func (*gkeAutopilot) MutateGameServerPod(gss *agonesv1.GameServerSpec, pod *corev1.Pod) error { - setPassthroughLabel(gss, pod) - setPrimaryContainer(pod, gss.Container) - podSpecSeccompUnconfined(&pod.Spec) - return nil -} - -// setPassthroughLabel sets the agones.dev/port: "autopilot-passthrough" label to the game server container. -// This will help to back the container port from the allocated port using an objectSelector of this label -// in GameServers that are using Passthrough Port Policy -func setPassthroughLabel(gs *agonesv1.GameServerSpec, pod *corev1.Pod) { - if hasPortPolicy(gs, agonesv1.Passthrough) { - pod.ObjectMeta.Labels[agonesv1.GameServerPortPolicyPodLabel] = "autopilot-passthrough" - } -} - -// setPrimaryContainer sets the autopilot.gke.io/primary-container annotation to the game server container. -// This acts as a hint to Autopilot for which container to add resources to during resource adjustment. -// See https://cloud.google.com/kubernetes-engine/docs/concepts/autopilot-resource-requests#autopilot-resource-management -// for more details. -func setPrimaryContainer(pod *corev1.Pod, containerName string) { - if _, ok := pod.ObjectMeta.Annotations[primaryContainerAnnotation]; ok { - return - } - pod.ObjectMeta.Annotations[primaryContainerAnnotation] = containerName -} - -// podSpecSeccompUnconfined sets to seccomp profile to `Unconfined` to avoid serious performance -// degradation possible with seccomp. We only set the pod level seccompProfile, and only set -// it if it hasn't been set - users can then override at either the pod or container level -// in the GameServer spec. -func podSpecSeccompUnconfined(podSpec *corev1.PodSpec) { - if podSpec.SecurityContext != nil && podSpec.SecurityContext.SeccompProfile != nil { - return - } - if podSpec.SecurityContext == nil { - podSpec.SecurityContext = &corev1.PodSecurityContext{} - } - podSpec.SecurityContext.SeccompProfile = &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeUnconfined} -} - -func (g *gkeAutopilot) SetEviction(ev *agonesv1.Eviction, pod *corev1.Pod) error { - if g.useExtendedDurationPods { - return eviction.SetEviction(ev, pod) - } - return setEvictionNoExtended(ev, pod) -} - -// setEvictionNoExtended sets disruption controls based on GameServer.Status.Eviction. For Autopilot: -// - Since the safe-to-evict pod annotation is not supported if "false", we delete it (if it's set -// to anything else, we allow it - Autopilot only rejects "false"). -// - OnUpgrade is not supported and rejected by validation above. Since we can't support -// safe-to-evict=false but can support a restrictive PDB, we can support Never and Always, but -// OnUpgrade doesn't make sense on Autopilot today. - an overly restrictive PDB prevents -// any sort of graceful eviction. -func setEvictionNoExtended(ev *agonesv1.Eviction, pod *corev1.Pod) error { - if safeAnnotation := pod.ObjectMeta.Annotations[agonesv1.PodSafeToEvictAnnotation]; safeAnnotation == agonesv1.False { - delete(pod.ObjectMeta.Annotations, agonesv1.PodSafeToEvictAnnotation) - } - if ev == nil { - return errors.New("No eviction value set. Should be the default value") - } - if _, exists := pod.ObjectMeta.Labels[agonesv1.SafeToEvictLabel]; !exists { - switch ev.Safe { - case agonesv1.EvictionSafeAlways: - // For EvictionSafeAlways, we use a label value that does not match the - // agones-gameserver-safe-to-evict-false PDB. But we go ahead and label - // it, in case someone wants to adopt custom logic for this group of - // game servers. - pod.ObjectMeta.Labels[agonesv1.SafeToEvictLabel] = agonesv1.True - case agonesv1.EvictionSafeNever: - pod.ObjectMeta.Labels[agonesv1.SafeToEvictLabel] = agonesv1.False - default: - return errors.Errorf("eviction.safe == %s, which webhook should have rejected on Autopilot", ev.Safe) - } - } - return nil -} - -func hasPortPolicy(gs *agonesv1.GameServerSpec, portPolicy agonesv1.PortPolicy) bool { - for _, p := range gs.Ports { - if p.PortPolicy == portPolicy { - return true - } - } - return false -} - -type autopilotPortAllocator struct { - minPort int32 - maxPort int32 -} - -func (*autopilotPortAllocator) Run(_ context.Context) error { return nil } -func (*autopilotPortAllocator) DeAllocate(_ *agonesv1.GameServer) {} - -func (apa *autopilotPortAllocator) Allocate(gs *agonesv1.GameServer) *agonesv1.GameServer { - if len(gs.Spec.Ports) == 0 { - return gs // Nothing to do. - } - - var ports []agonesv1.GameServerPort - for i, p := range gs.Spec.Ports { - if !(p.PortPolicy == agonesv1.Dynamic || p.PortPolicy == agonesv1.Passthrough) { - logger.WithField("gs", gs.Name).WithField("portPolicy", p.PortPolicy).Error( - "GameServer has invalid PortPolicy for Autopilot - this should have been rejected by webhooks. Refusing to assign ports.") - return gs - } - p.HostPort = int32(i + 1) // Autopilot expects _some_ host port - use a value unique to this GameServer Port. - - if p.Protocol == agonesv1.ProtocolTCPUDP { - tcp := p - tcp.Name = p.Name + "-tcp" - tcp.Protocol = corev1.ProtocolTCP - ports = append(ports, tcp) - - p.Name += "-udp" - p.Protocol = corev1.ProtocolUDP - } - ports = append(ports, p) - } - - hpa := hostPortAssignment{Min: apa.minPort, Max: apa.maxPort} - hpaJSON, err := json.Marshal(hpa) - if err != nil { - logger.WithError(err).WithField("hostPort", hpa).WithField("gs", gs.Name).Error("Internal error marshalling hostPortAssignment for GameServer") - // In error cases, return the original gs - on Autopilot this will result in a policy failure. - return gs - } - - // No errors past here. - gs.Spec.Ports = ports - if gs.Spec.Template.ObjectMeta.Annotations == nil { - gs.Spec.Template.ObjectMeta.Annotations = make(map[string]string) - } - gs.Spec.Template.ObjectMeta.Annotations[hostPortAssignmentAnnotation] = string(hpaJSON) - return gs -} - -// validatePortPolicy is a helper function to validate a single GameServerPort's PortPolicy -// for GKE Autopilot constraints. -func validatePortPolicy(p agonesv1.GameServerPort, i int, fldPath *field.Path) field.ErrorList { - var allErrs field.ErrorList - portPath := fldPath.Child("ports").Index(i) - - switch p.PortPolicy { - case agonesv1.Dynamic, agonesv1.Passthrough: - // These policies are always valid on GKE Autopilot. - case agonesv1.None: - // "None" is valid only if the feature gate FeaturePortPolicyNone is enabled. - if !runtime.FeatureEnabled(runtime.FeaturePortPolicyNone) { - allErrs = append(allErrs, field.Invalid(portPath.Child("portPolicy"), p.PortPolicy, "PortPolicy 'None' is not enabled")) - } - default: - // Any other port policy, such as "Static", is considered invalid on GKE Autopilot. - allErrs = append(allErrs, field.Invalid(portPath.Child("portPolicy"), p.PortPolicy, "portPolicy must be Dynamic, Passthrough, or None on GKE Autopilot")) - } - - if p.Range != agonesv1.DefaultPortRange && (p.PortPolicy != agonesv1.None || !runtime.FeatureEnabled(runtime.FeaturePortPolicyNone)) { - allErrs = append(allErrs, field.Invalid(fldPath.Child("ports").Index(i).Child("range"), p.Range, errRangeInvalid)) - } - - return allErrs -} +// Copyright Contributors to Agones a Series of LF Projects, LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gke implements the GKE cloud product (specifically Autopilot for now) +package gke + +import ( + "context" + "encoding/json" + "fmt" + + "agones.dev/agones/pkg/apis" + agonesv1 "agones.dev/agones/pkg/apis/agones/v1" + "agones.dev/agones/pkg/client/informers/externalversions" + "agones.dev/agones/pkg/cloudproduct/eviction" + "agones.dev/agones/pkg/portallocator" + "agones.dev/agones/pkg/util/runtime" + "cloud.google.com/go/compute/metadata" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" +) + +const ( + hostPortAssignmentAnnotation = "autopilot.gke.io/host-port-assignment" + primaryContainerAnnotation = "autopilot.gke.io/primary-container" + + errRangeInvalid = "range must not be used on GKE Autopilot" + errSchedulingMustBePacked = "scheduling strategy must be Packed on GKE Autopilot" + errEvictionSafeOnUpgradeInvalid = "eviction.safe OnUpgrade not supported on GKE Autopilot" +) + +var ( + autopilotMutatingWebhooks = []string{ + "workload-defaulter.config.common-webhooks.networking.gke.io", // pre-1.26 + "sasecret-redacter.config.common-webhooks.networking.gke.io", // 1.26+ + } + noWorkloadDefaulter = fmt.Sprintf("found no MutatingWebhookConfigurations matching %v", autopilotMutatingWebhooks) + + logger = runtime.NewLoggerWithSource("gke") +) + +type gkeAutopilot struct { + useExtendedDurationPods bool +} + +// hostPortAssignment is the JSON structure of the `host-port-assignment` annotation +// +//nolint:govet // API-like, keep consistent +type hostPortAssignment struct { + Min int32 `json:"min,omitempty"` + Max int32 `json:"max,omitempty"` + PortsAssigned map[int32]int32 `json:"portsAssigned,omitempty"` // old -> new +} + +// Detect whether we're running on GKE and/or Autopilot and return the appropriate +// cloud product string. +func Detect(ctx context.Context, kc *kubernetes.Clientset) string { + if !metadata.OnGCE() { + return "" + } + // Look for the workload defaulter - this is the current best method to detect Autopilot + found := false + for _, webhook := range autopilotMutatingWebhooks { + if _, err := kc.AdmissionregistrationV1().MutatingWebhookConfigurations().Get( + ctx, webhook, metav1.GetOptions{}); err != nil { + logger.WithError(err).WithField("webhook", webhook).Info("Detecting Autopilot MutatingWebhookConfiguration") + } else { + found = true + break + } + } + if !found { + logger.WithField("reason", noWorkloadDefaulter).Info( + "Assuming GKE Standard and defaulting to generic provider (expected if not on GKE Autopilot)") + return "" // GKE standard, but we don't need an interface for it just yet. + } + logger.Info("Running on GKE Autopilot (skip detection with --cloud-product=gke-autopilot)") + return "gke-autopilot" +} + +// Autopilot returns a GKE Autopilot cloud product +// +//nolint:revive // ignore the unexported return; implements ControllerHooksInterface +func Autopilot() *gkeAutopilot { + return &gkeAutopilot{useExtendedDurationPods: runtime.FeatureEnabled(runtime.FeatureGKEAutopilotExtendedDurationPods)} +} + +func (*gkeAutopilot) SyncPodPortsToGameServer(gs *agonesv1.GameServer, pod *corev1.Pod) error { + // If applyGameServerAddressAndPort has already filled in Status, SyncPodPortsToGameServer + // has already run. Skip syncing from the Pod again - this avoids having to reason + // about whether we're re-applying the old->new mapping. + if len(gs.Status.Ports) == len(gs.Spec.Ports) { + return nil + } + annotation, ok := pod.ObjectMeta.Annotations[hostPortAssignmentAnnotation] + if !ok { + return nil + } + var hpa hostPortAssignment + if err := json.Unmarshal([]byte(annotation), &hpa); err != nil { + return errors.Wrapf(err, "could not unmarshal annotation %s (value %q)", hostPortAssignmentAnnotation, annotation) + } + for i, p := range gs.Spec.Ports { + if newPort, ok := hpa.PortsAssigned[p.HostPort]; ok { + gs.Spec.Ports[i].HostPort = newPort + } + } + return nil +} + +func (*gkeAutopilot) NewPortAllocator(portRanges map[string]portallocator.PortRange, + _ informers.SharedInformerFactory, + _ externalversions.SharedInformerFactory, +) portallocator.Interface { + defPortRange := portRanges[agonesv1.DefaultPortRange] + return &autopilotPortAllocator{minPort: defPortRange.MinPort, maxPort: defPortRange.MaxPort} +} + +func (*gkeAutopilot) WaitOnFreePorts() bool { return true } + +func (g *gkeAutopilot) ValidateGameServerSpec(gss *agonesv1.GameServerSpec, fldPath *field.Path) field.ErrorList { + allErrs := g.ValidateScheduling(gss.Scheduling, fldPath.Child("scheduling")) + + // Loop through ports and use the helper function for validation + for i, p := range gss.Ports { + allErrs = append(allErrs, validatePortPolicy(p, i, fldPath)...) + + } + + // See SetEviction comment below for why we block EvictionSafeOnUpgrade, if Extended Duration pods aren't supported. + if !g.useExtendedDurationPods && gss.Eviction.Safe == agonesv1.EvictionSafeOnUpgrade { + allErrs = append(allErrs, field.Invalid(fldPath.Child("eviction").Child("safe"), string(gss.Eviction.Safe), errEvictionSafeOnUpgradeInvalid)) + } + return allErrs +} + +func (*gkeAutopilot) ValidateScheduling(ss apis.SchedulingStrategy, fldPath *field.Path) field.ErrorList { + if ss != apis.Packed { + return field.ErrorList{ + field.Invalid(fldPath, string(ss), errSchedulingMustBePacked), + } + } + return nil +} + +func (*gkeAutopilot) MutateGameServerPod(gss *agonesv1.GameServerSpec, pod *corev1.Pod) error { + setPassthroughLabel(gss, pod) + setPrimaryContainer(pod, gss.Container) + podSpecSeccompUnconfined(&pod.Spec) + return nil +} + +// setPassthroughLabel sets the agones.dev/port: "autopilot-passthrough" label to the game server container. +// This will help to back the container port from the allocated port using an objectSelector of this label +// in GameServers that are using Passthrough Port Policy +func setPassthroughLabel(gs *agonesv1.GameServerSpec, pod *corev1.Pod) { + if hasPortPolicy(gs, agonesv1.Passthrough) { + pod.ObjectMeta.Labels[agonesv1.GameServerPortPolicyPodLabel] = "autopilot-passthrough" + } +} + +// setPrimaryContainer sets the autopilot.gke.io/primary-container annotation to the game server container. +// This acts as a hint to Autopilot for which container to add resources to during resource adjustment. +// See https://cloud.google.com/kubernetes-engine/docs/concepts/autopilot-resource-requests#autopilot-resource-management +// for more details. +func setPrimaryContainer(pod *corev1.Pod, containerName string) { + if _, ok := pod.ObjectMeta.Annotations[primaryContainerAnnotation]; ok { + return + } + pod.ObjectMeta.Annotations[primaryContainerAnnotation] = containerName +} + +// podSpecSeccompUnconfined sets to seccomp profile to `Unconfined` to avoid serious performance +// degradation possible with seccomp. We only set the pod level seccompProfile, and only set +// it if it hasn't been set - users can then override at either the pod or container level +// in the GameServer spec. +func podSpecSeccompUnconfined(podSpec *corev1.PodSpec) { + if podSpec.SecurityContext != nil && podSpec.SecurityContext.SeccompProfile != nil { + return + } + if podSpec.SecurityContext == nil { + podSpec.SecurityContext = &corev1.PodSecurityContext{} + } + podSpec.SecurityContext.SeccompProfile = &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeUnconfined} +} + +func (g *gkeAutopilot) SetEviction(ev *agonesv1.Eviction, pod *corev1.Pod) error { + if g.useExtendedDurationPods { + return eviction.SetEviction(ev, pod) + } + return setEvictionNoExtended(ev, pod) +} + +// setEvictionNoExtended sets disruption controls based on GameServer.Status.Eviction. For Autopilot: +// - Since the safe-to-evict pod annotation is not supported if "false", we delete it (if it's set +// to anything else, we allow it - Autopilot only rejects "false"). +// - OnUpgrade is not supported and rejected by validation above. Since we can't support +// safe-to-evict=false but can support a restrictive PDB, we can support Never and Always, but +// OnUpgrade doesn't make sense on Autopilot today. - an overly restrictive PDB prevents +// any sort of graceful eviction. +func setEvictionNoExtended(ev *agonesv1.Eviction, pod *corev1.Pod) error { + if safeAnnotation := pod.ObjectMeta.Annotations[agonesv1.PodSafeToEvictAnnotation]; safeAnnotation == agonesv1.False { + delete(pod.ObjectMeta.Annotations, agonesv1.PodSafeToEvictAnnotation) + } + if ev == nil { + return errors.New("No eviction value set. Should be the default value") + } + if _, exists := pod.ObjectMeta.Labels[agonesv1.SafeToEvictLabel]; !exists { + switch ev.Safe { + case agonesv1.EvictionSafeAlways: + // For EvictionSafeAlways, we use a label value that does not match the + // agones-gameserver-safe-to-evict-false PDB. But we go ahead and label + // it, in case someone wants to adopt custom logic for this group of + // game servers. + pod.ObjectMeta.Labels[agonesv1.SafeToEvictLabel] = agonesv1.True + case agonesv1.EvictionSafeNever: + pod.ObjectMeta.Labels[agonesv1.SafeToEvictLabel] = agonesv1.False + default: + return errors.Errorf("eviction.safe == %s, which webhook should have rejected on Autopilot", ev.Safe) + } + } + return nil +} + +func hasPortPolicy(gs *agonesv1.GameServerSpec, portPolicy agonesv1.PortPolicy) bool { + for _, p := range gs.Ports { + if p.PortPolicy == portPolicy { + return true + } + } + return false +} + +type autopilotPortAllocator struct { + minPort int32 + maxPort int32 +} + +func (*autopilotPortAllocator) Run(_ context.Context) error { return nil } +func (*autopilotPortAllocator) DeAllocate(_ *agonesv1.GameServer) {} + +func (apa *autopilotPortAllocator) Allocate(gs *agonesv1.GameServer) *agonesv1.GameServer { + if len(gs.Spec.Ports) == 0 { + return gs // Nothing to do. + } + + var ports []agonesv1.GameServerPort + for i, p := range gs.Spec.Ports { + if !(p.PortPolicy == agonesv1.Dynamic || p.PortPolicy == agonesv1.Passthrough) { + logger.WithField("gs", gs.Name).WithField("portPolicy", p.PortPolicy).Error( + "GameServer has invalid PortPolicy for Autopilot - this should have been rejected by webhooks. Refusing to assign ports.") + return gs + } + p.HostPort = int32(i + 1) // Autopilot expects _some_ host port - use a value unique to this GameServer Port. + + if p.Protocol == agonesv1.ProtocolTCPUDP { + tcp := p + tcp.Name = p.Name + "-tcp" + tcp.Protocol = corev1.ProtocolTCP + ports = append(ports, tcp) + + p.Name += "-udp" + p.Protocol = corev1.ProtocolUDP + } + ports = append(ports, p) + } + + hpa := hostPortAssignment{Min: apa.minPort, Max: apa.maxPort} + hpaJSON, err := json.Marshal(hpa) + if err != nil { + logger.WithError(err).WithField("hostPort", hpa).WithField("gs", gs.Name).Error("Internal error marshalling hostPortAssignment for GameServer") + // In error cases, return the original gs - on Autopilot this will result in a policy failure. + return gs + } + + // No errors past here. + gs.Spec.Ports = ports + if gs.Spec.Template.ObjectMeta.Annotations == nil { + gs.Spec.Template.ObjectMeta.Annotations = make(map[string]string) + } + gs.Spec.Template.ObjectMeta.Annotations[hostPortAssignmentAnnotation] = string(hpaJSON) + return gs +} + +// validatePortPolicy is a helper function to validate a single GameServerPort's PortPolicy +// for GKE Autopilot constraints. +func validatePortPolicy(p agonesv1.GameServerPort, i int, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + portPath := fldPath.Child("ports").Index(i) + + switch p.PortPolicy { + case agonesv1.Dynamic, agonesv1.Passthrough: + // These policies are always valid on GKE Autopilot. + case agonesv1.None: + + default: + // Any other port policy, such as "Static", is considered invalid on GKE Autopilot. + allErrs = append(allErrs, field.Invalid(portPath.Child("portPolicy"), p.PortPolicy, "portPolicy must be Dynamic, Passthrough, or None on GKE Autopilot")) + } + + if p.Range != agonesv1.DefaultPortRange && p.PortPolicy != agonesv1.None { + allErrs = append(allErrs, field.Invalid(fldPath.Child("ports").Index(i).Child("range"), p.Range, errRangeInvalid)) + } + + return allErrs +} diff --git a/pkg/cloudproduct/gke/gke_test.go b/pkg/cloudproduct/gke/gke_test.go index 786aaa248b..3aaa3f6edb 100644 --- a/pkg/cloudproduct/gke/gke_test.go +++ b/pkg/cloudproduct/gke/gke_test.go @@ -84,7 +84,7 @@ func TestValidateGameServer(t *testing.T) { scheduling apis.SchedulingStrategy safeToEvict agonesv1.EvictionSafe want field.ErrorList - portPolicyNoneFlag string + }{ "no ports => validated": {scheduling: apis.Packed}, "good ports => validated": { @@ -254,31 +254,10 @@ func TestValidateGameServer(t *testing.T) { field.Invalid(field.NewPath("spec", "ports").Index(2).Child("portPolicy"), agonesv1.Static, "portPolicy must be Dynamic, Passthrough, or None on GKE Autopilot"), }, }, - "port policy none with feature disabled => fails validation": { - portPolicyNoneFlag: "false", - ports: []agonesv1.GameServerPort{ - { - Name: "none-gate-turned-off", - PortPolicy: agonesv1.None, - Range: agonesv1.DefaultPortRange, - ContainerPort: 1234, - Protocol: corev1.ProtocolUDP, - }, - }, - scheduling: apis.Packed, - want: field.ErrorList{ - field.Invalid(field.NewPath("spec", "ports").Index(0).Child("portPolicy"), agonesv1.None, "PortPolicy 'None' is not enabled"), - }, - }, + } { t.Run(name, func(t *testing.T) { - // PortPolicy None is behind a feature flag - runtime.FeatureTestMutex.Lock() - defer runtime.FeatureTestMutex.Unlock() - if tc.portPolicyNoneFlag == "" { - tc.portPolicyNoneFlag = "true" - } - require.NoError(t, runtime.ParseFeatures(fmt.Sprintf("%s=%s", runtime.FeaturePortPolicyNone, tc.portPolicyNoneFlag))) + causes := (&gkeAutopilot{useExtendedDurationPods: tc.edPods}).ValidateGameServerSpec(&agonesv1.GameServerSpec{ Ports: tc.ports, Scheduling: tc.scheduling, diff --git a/pkg/gameservers/gameservers_test.go b/pkg/gameservers/gameservers_test.go index a36a1ae928..33b4ff7857 100644 --- a/pkg/gameservers/gameservers_test.go +++ b/pkg/gameservers/gameservers_test.go @@ -166,10 +166,6 @@ func TestApplyGameServerAddressAndPort(t *testing.T) { pod.Status.PodIPs = []corev1.PodIP{{IP: ipFixture}} tc.podMod(pod) - // PortPolicy None is behind a feature flag - runtime.FeatureTestMutex.Lock() - defer runtime.FeatureTestMutex.Unlock() - require.NoError(t, runtime.ParseFeatures(string(runtime.FeaturePortPolicyNone)+"=true")) gs, err := applyGameServerAddressAndPort(gsFixture, node, pod, tc.podSyncer) require.NoError(t, err) diff --git a/pkg/util/runtime/features.go b/pkg/util/runtime/features.go index 32fcc8c73b..5717f206aa 100644 --- a/pkg/util/runtime/features.go +++ b/pkg/util/runtime/features.go @@ -1,281 +1,281 @@ -// Copyright Contributors to Agones a Series of LF Projects, LLC. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package runtime - -import ( - "net/url" - "strconv" - "sync" - - "github.com/pkg/errors" - "github.com/spf13/pflag" - "github.com/spf13/viper" -) - -const ( - // FeatureGateFlag is a name of a command line flag, which turns on specific tests for FeatureGates - FeatureGateFlag = "feature-gates" - - //////////////// - // Stable features - - // FeatureDisableResyncOnSDKServer is a feature flag to enable/disable resync on SDK server. - FeatureDisableResyncOnSDKServer Feature = "DisableResyncOnSDKServer" - - // FeatureAutopilotPassthroughPort is a feature flag that enables/disables Passthrough Port Policy. - FeatureAutopilotPassthroughPort Feature = "AutopilotPassthroughPort" - - //////////////// - // Beta features - - // FeatureCountsAndLists is a feature flag that enables counts and lists feature - // (a generic implenetation of the player tracking feature). - FeatureCountsAndLists Feature = "CountsAndLists" - - // FeatureGKEAutopilotExtendedDurationPods enables the use of Extended Duration pods - // when Agones is running on Autopilot. Available on 1.28+ only. - FeatureGKEAutopilotExtendedDurationPods = "GKEAutopilotExtendedDurationPods" - - // FeaturePortPolicyNone is a feature flag to allow setting Port Policy to None. - FeaturePortPolicyNone Feature = "PortPolicyNone" - - // FeaturePortRanges is a feature flag to enable/disable specific port ranges. - FeaturePortRanges Feature = "PortRanges" - - // FeatureRollingUpdateFix is a feature flag to enable/disable fleet controller fixes. - FeatureRollingUpdateFix Feature = "RollingUpdateFix" - - // FeatureScheduledAutoscaler is a feature flag to enable/disable scheduled fleet autoscaling. - FeatureScheduledAutoscaler Feature = "ScheduledAutoscaler" - - // FeatureSidecarContainers is a feature flag to enable/disable k8s sidecar containers for the sdkserver - FeatureSidecarContainers = "SidecarContainers" - - // FeatureFleetAutoscaleRequestMetaData is a feature flag that enables/disables fleet metadata on webhook autoscaler request. - FeatureFleetAutoscaleRequestMetaData Feature = "FleetAutoscaleRequestMetaData" - - //////////////// - // Alpha features - - // FeaturePlayerAllocationFilter is a feature flag that enables the ability for Allocations to filter based on - // player capacity. - FeaturePlayerAllocationFilter Feature = "PlayerAllocationFilter" - - // FeaturePlayerTracking is a feature flag to enable/disable player tracking features. - FeaturePlayerTracking Feature = "PlayerTracking" - - // FeatureWasmAutoscaler is a feature flag to enable/disable the script based autoscaler. - FeatureWasmAutoscaler Feature = "WasmAutoscaler" - - //////////////// - // Dev features - - // FeatureProcessorAllocator is a feature flag to enable/disable the processor allocator feature. - FeatureProcessorAllocator = "ProcessorAllocator" - - //////////////// - // Example feature - - // FeatureExample is an example feature gate flag, used for testing and demonstrative purposes - FeatureExample Feature = "Example" -) - -var ( - // featureDefaults is a map of all Feature Gates that are - // operational in Agones, and what their default configuration is. - // dev & alpha features are disabled by default; beta features are enabled. - // - // To add a new dev feature (an in progress feature, not tested in CI and not publicly documented): - // * add a const above - // * add it to `featureDefaults` - // * add it to install/helm/agones/defaultfeaturegates.yaml - // * note: you can add a new feature as an alpha feature if you're ready to test it in CI - // - // To promote a feature from dev->alpha: - // * add it to `ALPHA_FEATURE_GATES` in build/Makefile - // * add the inverse to the e2e-runner config in cloudbuild.yaml - // * add it to site/content/en/docs/Guides/feature-stages.md - // * add it to test/upgrade/versionMap.yaml - // * Ensure that the features in each file are organized categorically and alphabetically. - // - // To promote a feature from alpha->beta: - // * move from `false` to `true` in `featureDefaults`. - // * move from `false` to `true` in install/helm/agones/defaultfeaturegates.yaml - // * remove from `ALPHA_FEATURE_GATES` in build/Makefile - // * add to `BETA_FEATURE_GATES` in build/Makefile - // * invert in the e2e-runner config in cloudbuild.yaml - // * change the value in site/content/en/docs/Guides/feature-stages.md. - // * add it to test/upgrade/versionMap.yaml - // * Ensure that the features in each file are organized categorically and alphabetically. - // - // Feature Promotion: alpha->beta for SDK Functions - // * Move methods from alpha->beta files: - // - From proto/sdk/alpha/alpha.proto to proto/sdk/beta/beta.proto - // - For each language-specific SDK (e.g., Go, C#, Rust): - // - Move implementation files (e.g., alpha.go to beta.go) - // - Move test files (e.g., alpha_test.go to beta_test.go) - // - Note: Delete references to 'alpha' in the moved alpha methods. - // * Change all code and documentation references of alpha->beta: - // - Proto Files: proto/sdk/sdk.proto `[Stage:Alpha]->[Stage:Beta]` - // - SDK Implementations: Update in language-specific SDKs (e.g., sdks/go/sdk.go, sdks/csharp/sdk/AgonesSDK.cs). - // - Examples & Tests: Adjust in files like examples/simple-game-server/main.go and language-specific test files. - // * Modify automation scripts in the build/build-sdk-images directory to support beta file generation. - // * Run `make gen-all-sdk-grpc` to generate the required files. If there are changes to the `proto/allocation/allocation.proto` run `make gen-allocation-grpc`. - // * Afterwards, execute the `make run-sdk-conformance-test-go` command and address any issues that arise. - // * NOTE: DO NOT EDIT any autogenerated code. `make gen-all-sdk-grpc` will take care of it. - // - // To promote a feature from beta->GA: - // * Remove all places consuming the feature gate and fold logic to true. - // * Consider cleanup - often folding a gate to true allows refactoring. - // * Remove from the e2e-runner config in cloudbuild.yaml. - // * Update site/content/en/docs/Guides/feature-stages.md to indicate the feature is Stable. - // * Remove from `BETA_FEATURE_GATES` in build/Makefile. - // * Move from 'Beta features' in pkg/util/runtime/features.go and - // install/helm/agones/defaultfeaturegates.yaml to 'Stable features'. - // - // In each of these, keep the feature sorted by descending maturity then alphabetical - featureDefaults = map[Feature]bool{ - // Stable features - // Note that stable features cannot be set to "false", and are here so that upgrades from a - // previous version with the feature flag do not fail on parsing an unknown flag. - FeatureDisableResyncOnSDKServer: true, - FeatureAutopilotPassthroughPort: true, - - // Beta features - FeatureCountsAndLists: true, - FeatureGKEAutopilotExtendedDurationPods: true, - FeaturePortPolicyNone: true, - FeaturePortRanges: true, - FeatureRollingUpdateFix: true, - FeatureScheduledAutoscaler: true, - FeatureFleetAutoscaleRequestMetaData: true, - FeatureSidecarContainers: true, - - // Alpha features - FeaturePlayerAllocationFilter: false, - FeaturePlayerTracking: false, - FeatureWasmAutoscaler: false, - - // Dev features - FeatureProcessorAllocator: false, - - // Example feature - FeatureExample: false, - } - - // featureGates is the storage of what features are enabled - // or disabled. - featureGates map[Feature]bool - - // featureMutex ensures that updates to featureGates don't happen at the same time as reads. - // this is mostly to protect tests which can change gates in parallel. - featureMutex = sync.RWMutex{} - - // FeatureTestMutex is a mutex to be shared between tests to ensure that a test that involves changing featureGates - // cannot accidentally run at the same time as another test that also changing feature flags. - FeatureTestMutex sync.Mutex -) - -// Feature is a type for defining feature gates. -type Feature string - -// FeaturesBindFlags does the Viper arguments configuration. Call before running pflag.Parse() -func FeaturesBindFlags() { - viper.SetDefault(FeatureGateFlag, "") - pflag.String(FeatureGateFlag, viper.GetString(FeatureGateFlag), "Flag to pass in the url query list of feature flags to enable or disable") -} - -// FeaturesBindEnv binds the environment variables, based on the flags provided. -// call after viper.SetEnvKeyReplacer(...) if it is being set. -func FeaturesBindEnv() error { - return viper.BindEnv(FeatureGateFlag) -} - -// ParseFeaturesFromEnv will parse the feature flags from the Viper args -// configured by FeaturesBindFlags() and FeaturesBindEnv() -func ParseFeaturesFromEnv() error { - return ParseFeatures(viper.GetString(FeatureGateFlag)) -} - -// ParseFeatures parses the url encoded query string of features and stores the value -// for later retrieval -func ParseFeatures(queryString string) error { - featureMutex.Lock() - defer featureMutex.Unlock() - - features := map[Feature]bool{} - // copy the defaults into this map - for k, v := range featureDefaults { - features[k] = v - } - - values, err := url.ParseQuery(queryString) - if err != nil { - return errors.Wrap(err, "error parsing query string for feature gates") - } - - for k := range values { - f := Feature(k) - - if _, ok := featureDefaults[f]; !ok { - return errors.Errorf("Feature Gate %q is not a valid Feature Gate", f) - } - - b, err := strconv.ParseBool(values.Get(k)) - if err != nil { - return errors.Wrapf(err, "error parsing bool value from flag %s ", k) - } - features[f] = b - } - - featureGates = features - return nil -} - -// EnableAllFeatures turns on all feature flags. -// This is useful for libraries/processes/tests that want to -// enable all Alpha/Beta features without having to track all -// the current feature flags. -func EnableAllFeatures() { - featureMutex.Lock() - defer featureMutex.Unlock() - - features := map[Feature]bool{} - // copy the defaults into this map - for k := range featureDefaults { - features[k] = true - } - - featureGates = features -} - -// FeatureEnabled returns if a Feature is enabled or not -func FeatureEnabled(feature Feature) bool { - featureMutex.RLock() - defer featureMutex.RUnlock() - return featureGates[feature] -} - -// EncodeFeatures returns the feature set as a URL encoded query string -func EncodeFeatures() string { - values := url.Values{} - featureMutex.RLock() - defer featureMutex.RUnlock() - - for k, v := range featureGates { - values.Add(string(k), strconv.FormatBool(v)) - } - return values.Encode() -} +// Copyright Contributors to Agones a Series of LF Projects, LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runtime + +import ( + "net/url" + "strconv" + "sync" + + "github.com/pkg/errors" + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +const ( + // FeatureGateFlag is a name of a command line flag, which turns on specific tests for FeatureGates + FeatureGateFlag = "feature-gates" + + //////////////// + // Stable features + + // FeatureDisableResyncOnSDKServer is a feature flag to enable/disable resync on SDK server. + FeatureDisableResyncOnSDKServer Feature = "DisableResyncOnSDKServer" + + // FeatureAutopilotPassthroughPort is a feature flag that enables/disables Passthrough Port Policy. + FeatureAutopilotPassthroughPort Feature = "AutopilotPassthroughPort" + // FeaturePortPolicyNone is a feature flag to allow setting Port Policy to None. + FeaturePortPolicyNone Feature = "PortPolicyNone" + + //////////////// + // Beta features + + // FeatureCountsAndLists is a feature flag that enables counts and lists feature + // (a generic implenetation of the player tracking feature). + FeatureCountsAndLists Feature = "CountsAndLists" + + // FeatureGKEAutopilotExtendedDurationPods enables the use of Extended Duration pods + // when Agones is running on Autopilot. Available on 1.28+ only. + FeatureGKEAutopilotExtendedDurationPods = "GKEAutopilotExtendedDurationPods" + + + // FeaturePortRanges is a feature flag to enable/disable specific port ranges. + FeaturePortRanges Feature = "PortRanges" + + // FeatureRollingUpdateFix is a feature flag to enable/disable fleet controller fixes. + FeatureRollingUpdateFix Feature = "RollingUpdateFix" + + // FeatureScheduledAutoscaler is a feature flag to enable/disable scheduled fleet autoscaling. + FeatureScheduledAutoscaler Feature = "ScheduledAutoscaler" + + // FeatureSidecarContainers is a feature flag to enable/disable k8s sidecar containers for the sdkserver + FeatureSidecarContainers = "SidecarContainers" + + // FeatureFleetAutoscaleRequestMetaData is a feature flag that enables/disables fleet metadata on webhook autoscaler request. + FeatureFleetAutoscaleRequestMetaData Feature = "FleetAutoscaleRequestMetaData" + + //////////////// + // Alpha features + + // FeaturePlayerAllocationFilter is a feature flag that enables the ability for Allocations to filter based on + // player capacity. + FeaturePlayerAllocationFilter Feature = "PlayerAllocationFilter" + + // FeaturePlayerTracking is a feature flag to enable/disable player tracking features. + FeaturePlayerTracking Feature = "PlayerTracking" + + // FeatureWasmAutoscaler is a feature flag to enable/disable the script based autoscaler. + FeatureWasmAutoscaler Feature = "WasmAutoscaler" + + //////////////// + // Dev features + + // FeatureProcessorAllocator is a feature flag to enable/disable the processor allocator feature. + FeatureProcessorAllocator = "ProcessorAllocator" + + //////////////// + // Example feature + + // FeatureExample is an example feature gate flag, used for testing and demonstrative purposes + FeatureExample Feature = "Example" +) + +var ( + // featureDefaults is a map of all Feature Gates that are + // operational in Agones, and what their default configuration is. + // dev & alpha features are disabled by default; beta features are enabled. + // + // To add a new dev feature (an in progress feature, not tested in CI and not publicly documented): + // * add a const above + // * add it to `featureDefaults` + // * add it to install/helm/agones/defaultfeaturegates.yaml + // * note: you can add a new feature as an alpha feature if you're ready to test it in CI + // + // To promote a feature from dev->alpha: + // * add it to `ALPHA_FEATURE_GATES` in build/Makefile + // * add the inverse to the e2e-runner config in cloudbuild.yaml + // * add it to site/content/en/docs/Guides/feature-stages.md + // * add it to test/upgrade/versionMap.yaml + // * Ensure that the features in each file are organized categorically and alphabetically. + // + // To promote a feature from alpha->beta: + // * move from `false` to `true` in `featureDefaults`. + // * move from `false` to `true` in install/helm/agones/defaultfeaturegates.yaml + // * remove from `ALPHA_FEATURE_GATES` in build/Makefile + // * add to `BETA_FEATURE_GATES` in build/Makefile + // * invert in the e2e-runner config in cloudbuild.yaml + // * change the value in site/content/en/docs/Guides/feature-stages.md. + // * add it to test/upgrade/versionMap.yaml + // * Ensure that the features in each file are organized categorically and alphabetically. + // + // Feature Promotion: alpha->beta for SDK Functions + // * Move methods from alpha->beta files: + // - From proto/sdk/alpha/alpha.proto to proto/sdk/beta/beta.proto + // - For each language-specific SDK (e.g., Go, C#, Rust): + // - Move implementation files (e.g., alpha.go to beta.go) + // - Move test files (e.g., alpha_test.go to beta_test.go) + // - Note: Delete references to 'alpha' in the moved alpha methods. + // * Change all code and documentation references of alpha->beta: + // - Proto Files: proto/sdk/sdk.proto `[Stage:Alpha]->[Stage:Beta]` + // - SDK Implementations: Update in language-specific SDKs (e.g., sdks/go/sdk.go, sdks/csharp/sdk/AgonesSDK.cs). + // - Examples & Tests: Adjust in files like examples/simple-game-server/main.go and language-specific test files. + // * Modify automation scripts in the build/build-sdk-images directory to support beta file generation. + // * Run `make gen-all-sdk-grpc` to generate the required files. If there are changes to the `proto/allocation/allocation.proto` run `make gen-allocation-grpc`. + // * Afterwards, execute the `make run-sdk-conformance-test-go` command and address any issues that arise. + // * NOTE: DO NOT EDIT any autogenerated code. `make gen-all-sdk-grpc` will take care of it. + // + // To promote a feature from beta->GA: + // * Remove all places consuming the feature gate and fold logic to true. + // * Consider cleanup - often folding a gate to true allows refactoring. + // * Remove from the e2e-runner config in cloudbuild.yaml. + // * Update site/content/en/docs/Guides/feature-stages.md to indicate the feature is Stable. + // * Remove from `BETA_FEATURE_GATES` in build/Makefile. + // * Move from 'Beta features' in pkg/util/runtime/features.go and + // install/helm/agones/defaultfeaturegates.yaml to 'Stable features'. + // + // In each of these, keep the feature sorted by descending maturity then alphabetical + featureDefaults = map[Feature]bool{ + // Stable features + // Note that stable features cannot be set to "false", and are here so that upgrades from a + // previous version with the feature flag do not fail on parsing an unknown flag. + FeatureDisableResyncOnSDKServer: true, + FeatureAutopilotPassthroughPort: true, + + // Beta features + FeatureCountsAndLists: true, + FeatureGKEAutopilotExtendedDurationPods: true, + FeaturePortPolicyNone: true, + FeaturePortRanges: true, + FeatureRollingUpdateFix: true, + FeatureScheduledAutoscaler: true, + FeatureFleetAutoscaleRequestMetaData: true, + FeatureSidecarContainers: true, + + // Alpha features + FeaturePlayerAllocationFilter: false, + FeaturePlayerTracking: false, + FeatureWasmAutoscaler: false, + + // Dev features + FeatureProcessorAllocator: false, + + // Example feature + FeatureExample: false, + } + + // featureGates is the storage of what features are enabled + // or disabled. + featureGates map[Feature]bool + + // featureMutex ensures that updates to featureGates don't happen at the same time as reads. + // this is mostly to protect tests which can change gates in parallel. + featureMutex = sync.RWMutex{} + + // FeatureTestMutex is a mutex to be shared between tests to ensure that a test that involves changing featureGates + // cannot accidentally run at the same time as another test that also changing feature flags. + FeatureTestMutex sync.Mutex +) + +// Feature is a type for defining feature gates. +type Feature string + +// FeaturesBindFlags does the Viper arguments configuration. Call before running pflag.Parse() +func FeaturesBindFlags() { + viper.SetDefault(FeatureGateFlag, "") + pflag.String(FeatureGateFlag, viper.GetString(FeatureGateFlag), "Flag to pass in the url query list of feature flags to enable or disable") +} + +// FeaturesBindEnv binds the environment variables, based on the flags provided. +// call after viper.SetEnvKeyReplacer(...) if it is being set. +func FeaturesBindEnv() error { + return viper.BindEnv(FeatureGateFlag) +} + +// ParseFeaturesFromEnv will parse the feature flags from the Viper args +// configured by FeaturesBindFlags() and FeaturesBindEnv() +func ParseFeaturesFromEnv() error { + return ParseFeatures(viper.GetString(FeatureGateFlag)) +} + +// ParseFeatures parses the url encoded query string of features and stores the value +// for later retrieval +func ParseFeatures(queryString string) error { + featureMutex.Lock() + defer featureMutex.Unlock() + + features := map[Feature]bool{} + // copy the defaults into this map + for k, v := range featureDefaults { + features[k] = v + } + + values, err := url.ParseQuery(queryString) + if err != nil { + return errors.Wrap(err, "error parsing query string for feature gates") + } + + for k := range values { + f := Feature(k) + + if _, ok := featureDefaults[f]; !ok { + return errors.Errorf("Feature Gate %q is not a valid Feature Gate", f) + } + + b, err := strconv.ParseBool(values.Get(k)) + if err != nil { + return errors.Wrapf(err, "error parsing bool value from flag %s ", k) + } + features[f] = b + } + + featureGates = features + return nil +} + +// EnableAllFeatures turns on all feature flags. +// This is useful for libraries/processes/tests that want to +// enable all Alpha/Beta features without having to track all +// the current feature flags. +func EnableAllFeatures() { + featureMutex.Lock() + defer featureMutex.Unlock() + + features := map[Feature]bool{} + // copy the defaults into this map + for k := range featureDefaults { + features[k] = true + } + + featureGates = features +} + +// FeatureEnabled returns if a Feature is enabled or not +func FeatureEnabled(feature Feature) bool { + featureMutex.RLock() + defer featureMutex.RUnlock() + return featureGates[feature] +} + +// EncodeFeatures returns the feature set as a URL encoded query string +func EncodeFeatures() string { + values := url.Values{} + featureMutex.RLock() + defer featureMutex.RUnlock() + + for k, v := range featureGates { + values.Add(string(k), strconv.FormatBool(v)) + } + return values.Encode() +} diff --git a/site/content/en/docs/Guides/feature-stages.md b/site/content/en/docs/Guides/feature-stages.md index 12e5efee07..0258e231a4 100644 --- a/site/content/en/docs/Guides/feature-stages.md +++ b/site/content/en/docs/Guides/feature-stages.md @@ -41,7 +41,6 @@ The current set of `alpha`, `beta` and `stable` feature gates include: |------------------------------------------------------------------------------------------------------------------------|------------------------------------|---------|--------|--------| | [CountsAndLists](https://github.com/agones-dev/agones/issues/2716) | `CountsAndLists` | Enabled | `Beta` | 1.41.0 | | [Support for Extended Duration Pods on GKE Autopilot (*1.28+ only*)](https://github.com/agones-dev/agones/issues/3386) | `GKEAutopilotExtendedDurationPods` | Enabled | `Beta` | 1.44.0 | -| [Port Policy None](https://github.com/agones-dev/agones/issues/3804) | `PortPolicyNone` | Enabled | `Beta` | 1.49.0 | | [Multiple dynamic port ranges](https://github.com/agones-dev/agones/issues/1911) | `PortRanges` | Enabled | `Beta` | 1.49.0 | | [Rolling Update Fixes](https://github.com/agones-dev/agones/issues/3688) | `RollingUpdateFix` | Enabled | `Beta` | 1.50.0 | | [Scheduled Fleet Autoscaling](https://github.com/agones-dev/agones/issues/3008) | `ScheduledAutoscaler` | Enabled | `Beta` | 1.51.0 | @@ -54,6 +53,7 @@ The current set of `alpha`, `beta` and `stable` feature gates include: |------------------------------------------------------------------------------------------|----------------------------|---------|----------|--------| | [DisableResyncOnSDKServer](https://github.com/agones-dev/agones/issues/3377) | `DisableResyncOnSDKServer` | Enabled | `Stable` | 1.49.0 | | [Support Passthrough on GKE Autopilot](https://github.com/agones-dev/agones/issues/3721) | `AutopilotPassthroughPort` | Enabled | `Stable` | 1.54.0 | +| [Port Policy None](https://github.com/agones-dev/agones/issues/3804) | `PortPolicyNone` | Enabled | `Stable` | 1.49.0 | [fleet-updates]: {{% relref "./fleet-updates.md#notifying-gameservers-on-fleet-updatedownscale" %}} diff --git a/test/e2e/gameserver_test.go b/test/e2e/gameserver_test.go index 1757fdccbe..4517ee7b7e 100644 --- a/test/e2e/gameserver_test.go +++ b/test/e2e/gameserver_test.go @@ -1,2011 +1,2008 @@ -// Copyright Contributors to Agones a Series of LF Projects, LLC. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package e2e - -import ( - "bytes" - "context" - "fmt" - "net" - "os" - "os/exec" - "sort" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - policyv1 "k8s.io/api/policy/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/rand" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/utils/ptr" - - agonesv1 "agones.dev/agones/pkg/apis/agones/v1" - allocationv1 "agones.dev/agones/pkg/apis/allocation/v1" - agtesting "agones.dev/agones/pkg/testing" - "agones.dev/agones/pkg/util/runtime" - e2eframework "agones.dev/agones/test/e2e/framework" -) - -const ( - fakeIPAddress = "192.1.1.2" -) - -func TestCreateConnect(t *testing.T) { - t.Parallel() - ctx := context.Background() - gs := framework.DefaultGameServer(framework.Namespace) - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - if err != nil { - t.Fatalf("Could not get a GameServer ready: %v", err) - } - assert.Equal(t, len(readyGs.Status.Ports), 1) - assert.NotEmpty(t, readyGs.Status.Ports[0].Port) - assert.NotEmpty(t, readyGs.Status.Address) - assert.NotEmpty(t, readyGs.Status.Addresses) - - require.NotEmpty(t, readyGs.Status.NodeName) - require.Equal(t, readyGs.Status.State, agonesv1.GameServerStateReady) - - // check connectivity before anything else. - reply, err := framework.SendGameServerUDP(t, readyGs, "Hello World !") - require.NoError(t, err) - require.Equal(t, "ACK: Hello World !\n", reply) - - // since the PodIP could come at any point, let's eventually it. - require.EventuallyWithT(t, func(c *assert.CollectT) { - readyGs, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) - require.NoError(c, err) - for i, addr := range readyGs.Status.Addresses { - if addr.Type == agonesv1.NodePodIP { - require.NotEmpty(c, readyGs.Status.Addresses[i].Address) - } - } - }, 3*time.Minute, time.Second, "Failed to get PodIP") -} - -func TestHostName(t *testing.T) { - t.Parallel() - - pods := framework.KubeClient.CoreV1().Pods(framework.Namespace) - - fixtures := map[string]struct { - setup func(gs *agonesv1.GameServer) - test func(gs *agonesv1.GameServer, pod *corev1.Pod) - }{ - "standard hostname": { - setup: func(_ *agonesv1.GameServer) {}, - test: func(gs *agonesv1.GameServer, pod *corev1.Pod) { - assert.Equal(t, gs.ObjectMeta.Name, pod.Spec.Hostname) - }, - }, - "a . in the name": { - setup: func(gs *agonesv1.GameServer) { - gs.ObjectMeta.GenerateName = "game-server-1.0-" - }, - test: func(_ *agonesv1.GameServer, pod *corev1.Pod) { - expected := "game-server-1-0-" - // since it's a generated name, we just check the beginning. - assert.Equal(t, expected, pod.Spec.Hostname[:len(expected)]) - }, - }, - // generated name will automatically truncate to 63 chars. - "generated with > 63 chars": { - setup: func(gs *agonesv1.GameServer) { - gs.ObjectMeta.GenerateName = "game-server-" + strings.Repeat("n", 100) - }, - test: func(gs *agonesv1.GameServer, pod *corev1.Pod) { - assert.Equal(t, gs.ObjectMeta.Name, pod.Spec.Hostname) - }, - }, - // Note: no need to test for a gs.ObjectMeta.Name > 63 chars, as it will be rejected as invalid - } - - for k, v := range fixtures { - t.Run(k, func(t *testing.T) { - gs := framework.DefaultGameServer(framework.Namespace) - gs.Spec.Template.Spec.Subdomain = "default" - v.setup(gs) - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - require.NoError(t, err) - pod, err := pods.Get(context.Background(), readyGs.ObjectMeta.Name, metav1.GetOptions{}) - require.NoError(t, err) - v.test(readyGs, pod) - }) - } -} - -// nolint:dupl -func TestSDKSetLabel(t *testing.T) { - t.Parallel() - ctx := context.Background() - gs := framework.DefaultGameServer(framework.Namespace) - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - if err != nil { - t.Fatalf("Could not get a GameServer ready: %v", err) - } - - assert.Equal(t, readyGs.Status.State, agonesv1.GameServerStateReady) - reply, err := framework.SendGameServerUDP(t, readyGs, "LABEL") - if err != nil { - t.Fatalf("Could ping GameServer: %v", err) - } - - assert.Equal(t, "ACK: LABEL\n", reply) - - // the label is set in a queue, so it may take a moment - err = wait.PollUntilContextTimeout(ctx, time.Second, 10*time.Second, true, func(ctx context.Context) (bool, error) { - gs, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) - if err != nil { - return true, err - } - return gs.ObjectMeta.Labels != nil, nil - }) - - if assert.NoError(t, err) { - assert.NotEmpty(t, gs.ObjectMeta.Labels["agones.dev/sdk-timestamp"]) - } -} - -func TestHealthCheckDisable(t *testing.T) { - t.Parallel() - ctx := context.Background() - gs := framework.DefaultGameServer(framework.Namespace) - gs.Spec.Health = agonesv1.Health{ - Disabled: true, - FailureThreshold: 1, - InitialDelaySeconds: 1, - PeriodSeconds: 1, - } - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - if err != nil { - t.Fatalf("Could not get a GameServer ready: %v", err) - } - defer framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck - - _, err = framework.SendGameServerUDP(t, readyGs, "UNHEALTHY") - if err != nil { - t.Fatalf("Could not ping GameServer: %v", err) - } - - time.Sleep(10 * time.Second) - - gs, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) - if err != nil { - assert.FailNow(t, "gameserver get failed", err.Error()) - } - - assert.Equal(t, agonesv1.GameServerStateReady, gs.Status.State) -} - -// nolint:dupl -func TestSDKSetAnnotation(t *testing.T) { - t.Parallel() - ctx := context.Background() - gs := framework.DefaultGameServer(framework.Namespace) - annotation := "agones.dev/sdk-timestamp" - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - if err != nil { - t.Fatalf("Could not get a GameServer ready: %v", err) - } - defer framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck - - assert.Equal(t, readyGs.Status.State, agonesv1.GameServerStateReady) - reply, err := framework.SendGameServerUDP(t, readyGs, "ANNOTATION") - if err != nil { - t.Fatalf("Could ping GameServer: %v", err) - } - - assert.Equal(t, "ACK: ANNOTATION\n", reply) - - // the label is set in a queue, so it may take a moment - err = wait.PollUntilContextTimeout(context.Background(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { - gs, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) - if err != nil { - return true, err - } - - _, ok := gs.ObjectMeta.Annotations[annotation] - return ok, nil - }) - - logrus.WithField("annotations", gs.ObjectMeta.Annotations).Info("annotation information") - - if !assert.Nil(t, err) { - assert.FailNow(t, "error waiting on annotation to be set") - } - assert.NotEmpty(t, gs.ObjectMeta.Annotations[annotation]) - assert.NotEmpty(t, gs.ObjectMeta.Annotations[agonesv1.VersionAnnotation]) -} - -func TestUnhealthyGameServerAfterHealthCheckFail(t *testing.T) { - t.Parallel() - gs := framework.DefaultGameServer(framework.Namespace) - gs.Spec.Health.FailureThreshold = 1 - - gs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - if err != nil { - assert.FailNow(t, "Failed to create a gameserver", err.Error()) - } - - reply, err := framework.SendGameServerUDP(t, gs, "UNHEALTHY") - if err != nil { - assert.FailNow(t, "Failed to send a message to a gameserver", err.Error()) - } - assert.Equal(t, "ACK: UNHEALTHY\n", reply) - - _, err = framework.WaitForGameServerState(t, gs, agonesv1.GameServerStateUnhealthy, time.Minute) - assert.NoError(t, err) -} - -func TestUnhealthyGameServersWithoutFreePorts(t *testing.T) { - framework.SkipOnCloudProduct(t, "gke-autopilot", "does not support Static PortPolicy") - if runtime.FeatureEnabled(runtime.FeatureSidecarContainers) { - t.SkipNow() - } - - t.Parallel() - ctx := context.Background() - nodes, err := framework.KubeClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) - if err != nil { - assert.FailNow(t, "Failed to list nodes", err.Error()) - } - assert.True(t, len(nodes.Items) > 0) - - template := framework.DefaultGameServer(framework.Namespace) - // choose port out of the minport/maxport range - // also rand it, just in case there are still previous static GameServers floating around. - template.Spec.Ports[0].HostPort = int32(rand.IntnRange(4000, 5000)) - template.Spec.Ports[0].PortPolicy = agonesv1.Static - - gameServers := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace) - // one successful static port GameServer - gs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, template.DeepCopy()) - require.NoError(t, err) - - // now let's create the same one, but this time, require it be on the same node. - newGs := template.DeepCopy() - - if newGs.Spec.Template.Spec.NodeSelector == nil { - newGs.Spec.Template.Spec.NodeSelector = map[string]string{} - } - newGs.Spec.Template.Spec.NodeSelector["kubernetes.io/hostname"] = gs.Status.NodeName - newGs, err = gameServers.Create(ctx, newGs, metav1.CreateOptions{}) - require.NoError(t, err) - - _, err = framework.WaitForGameServerState(t, newGs, agonesv1.GameServerStateUnhealthy, 5*time.Minute) - assert.NoError(t, err) -} - -func TestGameServerUnhealthyAfterDeletingPod(t *testing.T) { - t.Parallel() - ctx := context.Background() - gs := framework.DefaultGameServer(framework.Namespace) - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - if err != nil { - t.Fatalf("Could not get a GameServer ready: %v", err) - } - - logrus.WithField("gsKey", readyGs.ObjectMeta.Name).Info("GameServer Ready") - - gsClient := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace) - podClient := framework.KubeClient.CoreV1().Pods(framework.Namespace) - - defer gsClient.Delete(ctx, readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck - - pod, err := podClient.Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) - require.NoError(t, err) - require.True(t, metav1.IsControlledBy(pod, readyGs)) - - err = podClient.Delete(ctx, pod.ObjectMeta.Name, metav1.DeleteOptions{}) - require.NoError(t, err) - - _, err = framework.WaitForGameServerState(t, readyGs, agonesv1.GameServerStateUnhealthy, 3*time.Minute) - require.NoError(t, err) -} - -func TestGameServerRestartBeforeReadyCrash(t *testing.T) { - if runtime.FeatureEnabled(runtime.FeatureSidecarContainers) { - t.SkipNow() - } - - t.Parallel() - ctx := context.Background() - logger := e2eframework.TestLogger(t) - - gs := framework.DefaultGameServer(framework.Namespace) - // give some buffer with gameservers crashing and coming back - gs.Spec.Health.PeriodSeconds = 60 * 60 - gs.Spec.Template.Spec.Containers[0].Env = append(gs.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "READY", Value: "FALSE"}) - gsClient := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace) - newGs, err := gsClient.Create(ctx, gs, metav1.CreateOptions{}) - if err != nil { - assert.Fail(t, "could not create the gameserver", err.Error()) - } - defer gsClient.Delete(ctx, newGs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck - - logger.Info("Waiting for us to have an address to send things to") - newGs, err = framework.WaitForGameServerState(t, newGs, agonesv1.GameServerStateScheduled, framework.WaitForState) - if err != nil { - assert.FailNow(t, "Failed schedule a pod", err.Error()) - } - - logger.WithField("gs", newGs.ObjectMeta.Name).Info("GameServer created") - - address := fmt.Sprintf("%s:%d", newGs.Status.Address, newGs.Status.Ports[0].Port) - logger.WithField("address", address).Info("Dialing UDP message to address") - - messageAndWait := func(gs *agonesv1.GameServer, msg string, check func(gs *agonesv1.GameServer, pod *corev1.Pod) bool) error { - return wait.PollUntilContextTimeout(context.Background(), 200*time.Millisecond, 3*time.Minute, true, func(ctx context.Context) (bool, error) { - gs, err := gsClient.Get(ctx, gs.ObjectMeta.Name, metav1.GetOptions{}) - if err != nil { - logger.WithError(err).Warn("could not get gameserver") - return true, err - } - pod, err := framework.KubeClient.CoreV1().Pods(framework.Namespace).Get(ctx, newGs.ObjectMeta.Name, metav1.GetOptions{}) - if err != nil { - logger.WithError(err).Warn("could not get pod for gameserver") - return true, err - } - - if check(gs, pod) { - return true, nil - } - - // create a connection each time, as weird stuff happens if the receiver isn't up and running. - conn, err := net.Dial("udp", address) - if err != nil { - logger.WithError(err).Warn("could not create connection") - return true, err - } - defer conn.Close() // nolint: errcheck - // doing this last, so that there is a short delay between the msg being sent, and the check. - logger.WithField("gs", gs.ObjectMeta.Name).WithField("msg", msg). - WithField("state", gs.Status.State).Info("sending message") - if _, err = conn.Write([]byte(msg)); err != nil { - logger.WithError(err).WithField("gs", gs.ObjectMeta.Name). - WithField("state", gs.Status.State).Info("error sending packet") - } - return false, nil - }) - } - - logger.Info("crashing, and waiting to see restart") - err = messageAndWait(newGs, "CRASH", func(_ *agonesv1.GameServer, pod *corev1.Pod) bool { - for _, c := range pod.Status.ContainerStatuses { - if c.Name == newGs.Spec.Container && c.RestartCount > 0 { - logger.Info("successfully crashed. Moving on!") - return true - } - } - return false - }) - assert.NoError(t, err) - - // check that the GameServer is not in an unhealthy state. If it does happen, it should happen pretty quick. - // We wait an extra 5s to close the kubelet race in #2445. - newGs, err = framework.WaitForGameServerState(t, newGs, agonesv1.GameServerStateUnhealthy, 10*time.Second) - // should be an error, as the state should not occur - if !assert.Error(t, err) { - assert.FailNow(t, "GameServer should not be Unhealthy") - } - assert.Contains(t, err.Error(), "waiting for GameServer") - - // ping READY until it doesn't fail anymore - since it may take a while - // for this to come back up -- or we could get a delayed CRASH, so we have to - // wait for the process to restart again to fire the SDK.Ready() - logger.Info("marking GameServer as ready") - err = messageAndWait(newGs, "READY", func(gs *agonesv1.GameServer, _ *corev1.Pod) bool { - if gs.Status.State == agonesv1.GameServerStateReady { - logger.Info("ready! Moving On!") - return true - } - return false - }) - assert.NoError(t, err) - - // now crash, should be unhealthy, since it's after being Ready - logger.Info("crashing again, should be unhealthy") - // retry on crash, as with the restarts, sometimes Go takes a moment to send this through. - err = messageAndWait(newGs, "CRASH", func(gs *agonesv1.GameServer, _ *corev1.Pod) bool { - logger.WithField("gs", gs.ObjectMeta.Name).WithField("state", gs.Status.State). - Info("checking final crash state") - if gs.Status.State == agonesv1.GameServerStateUnhealthy { - logger.Info("Unhealthy! We are done!") - return true - } - return false - }) - assert.NoError(t, err) -} - -func TestGameServerUnhealthyAfterReadyCrash(t *testing.T) { - t.Parallel() - ctx := context.Background() - log := e2eframework.TestLogger(t) - - gs := framework.DefaultGameServer(framework.Namespace) - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - if err != nil { - t.Fatalf("Could not get a GameServer ready: %v", err) - } - - log.WithField("gs", readyGs.ObjectMeta.Name).Info("GameServer created") - - gsClient := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace) - defer gsClient.Delete(ctx, readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck - - address := fmt.Sprintf("%s:%d", readyGs.Status.Address, readyGs.Status.Ports[0].Port) - - // keep crashing, until we move to Unhealthy. Solves potential issues with controller Informer cache - // race conditions in which it has yet to see a GameServer is Ready before the crash. - var stop int32 - defer func() { - atomic.StoreInt32(&stop, 1) - }() - go func() { - for { - if atomic.LoadInt32(&stop) > 0 { - log.Info("UDP Crash stop signal received. Stopping.") - return - } - var writeErr error - func() { - conn, err := net.Dial("udp", address) - assert.NoError(t, err) - defer conn.Close() // nolint: errcheck - _, writeErr = conn.Write([]byte("CRASH")) - }() - if writeErr != nil { - log.WithError(err).Warn("error sending udp packet. Stopping.") - return - } - log.WithField("address", address).Info("sent UDP packet") - time.Sleep(5 * time.Second) - } - }() - _, err = framework.WaitForGameServerState(t, readyGs, agonesv1.GameServerStateUnhealthy, 3*time.Minute) - assert.NoError(t, err) -} - -func TestGameServerPodCompletedAfterCleanExit(t *testing.T) { - if !runtime.FeatureEnabled(runtime.FeatureSidecarContainers) { - t.SkipNow() - } - - t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - log := e2eframework.TestLogger(t) - - gs := framework.DefaultGameServer(framework.Namespace) - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - if err != nil { - t.Fatalf("Could not get a GameServer ready: %v", err) - } - - address := fmt.Sprintf("%s:%d", readyGs.Status.Address, readyGs.Status.Ports[0].Port) - conn, err := net.Dial("udp", address) - mtx := &sync.Mutex{} - require.NoError(t, err) - defer func() { - mtx.Lock() - defer mtx.Unlock() - if conn != nil { - err = conn.Close() - if err != nil { - log.WithError(err).Warn("error closing udp connection") - } - } - }() - - go func() { - for { - select { - case <-ctx.Done(): - log.Info("Stopping udp CRASH sender") - return - default: - mtx.Lock() - log.Info("Sending CRASH 0") - _, err := conn.Write([]byte("CRASH 0")) - mtx.Unlock() - if err != nil { - log.WithError(err).Warn("error sending udp packet.") - } - } - time.Sleep(5 * time.Second) - } - }() - - result := assert.EventuallyWithT(t, func(c *assert.CollectT) { - gs, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) - log.WithField("gs", readyGs.ObjectMeta.Name).WithField("deletionTimestamp", gs.ObjectMeta.DeletionTimestamp).WithField("state", gs.Status.State).WithError(err).Info("checking if GameServer exists") - assert.True(c, k8serrors.IsNotFound(err)) - }, 5*time.Minute, 3*time.Second) - // debug a lot if it doesn't work, because why! - if !result { - framework.LogEvents(t, log, readyGs.ObjectMeta.Namespace, readyGs) - pod, err := framework.KubeClient.CoreV1().Pods(readyGs.ObjectMeta.Namespace).Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) - if err != nil { - log.WithError(err).Warn("error getting pod for GameServer, skipping debug output") - return - } - log.WithField("metadata", pod.ObjectMeta).WithField("status", pod.Status) - framework.LogEvents(t, log, readyGs.ObjectMeta.Namespace, pod) - framework.LogPodContainers(t, pod) - } -} - -func TestDevelopmentGameServerLifecycle(t *testing.T) { - t.Parallel() - ctx := context.Background() - - labels := map[string]string{"development": "true"} - gs := &agonesv1.GameServer{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "udp-server", - Namespace: framework.Namespace, - Annotations: map[string]string{agonesv1.DevAddressAnnotation: fakeIPAddress}, - Labels: labels, - }, - Spec: agonesv1.GameServerSpec{ - Container: "udp-server", - Ports: []agonesv1.GameServerPort{{ - ContainerPort: 7654, - HostPort: 7654, - Name: "gameport", - PortPolicy: agonesv1.Static, - Protocol: corev1.ProtocolUDP, - }}, - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{ - Name: "placebo", - Image: "this is ignored", - }}, - }, - }, - }, - } - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs.DeepCopy()) - if err != nil { - t.Fatalf("Could not get a GameServer ready: %v", err) - } - require.Equal(t, agonesv1.GameServerStateReady, readyGs.Status.State) - - // Set dev GS into RequestReady and confirm it goes back to Ready. - gsCopy := readyGs.DeepCopy() - gsCopy.Status.State = agonesv1.GameServerStateRequestReady - reqReadyGs, err := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Update(ctx, gsCopy, metav1.UpdateOptions{}) - require.NoError(t, err) - require.Equal(t, agonesv1.GameServerStateRequestReady, reqReadyGs.Status.State) - - readyGs, err = framework.WaitForGameServerState(t, reqReadyGs, agonesv1.GameServerStateReady, framework.WaitForState) - if err != nil { - t.Fatalf("Could not get a GameServer ready from request ready: %v", err) - } - require.Equal(t, agonesv1.GameServerStateReady, readyGs.Status.State) - - // confirm delete works, because if the finalisers don't get removed, this won't work. - err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) - require.NoError(t, err) - - err = wait.PollUntilContextTimeout(context.Background(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { - _, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) - if k8serrors.IsNotFound(err) { - return true, nil - } - - return false, err - }) - require.NoError(t, err) - - // let's make sure we can allocate a dev gameserver - readyGs, err = framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs.DeepCopy()) - require.NoError(t, err) - - gsa := &allocationv1.GameServerAllocation{ - Spec: allocationv1.GameServerAllocationSpec{ - Selectors: []allocationv1.GameServerSelector{{ - LabelSelector: metav1.LabelSelector{MatchLabels: labels}, - }}, - }, - } - gsa, err = framework.AgonesClient.AllocationV1().GameServerAllocations(framework.Namespace).Create(ctx, gsa, metav1.CreateOptions{}) - require.NoError(t, err) - - require.Equal(t, readyGs.ObjectMeta.Name, gsa.Status.GameServerName) - - _, err = framework.WaitForGameServerState(t, readyGs, agonesv1.GameServerStateAllocated, time.Minute) - require.NoError(t, err) - - // Also confirm that delete works for Allocated state, it should be fine but let's be sure. - err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) - require.NoError(t, err) - - err = wait.PollUntilContextTimeout(context.Background(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { - _, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) - if k8serrors.IsNotFound(err) { - return true, nil - } - - return false, err - }) - require.NoError(t, err) -} - -func TestGameServerSelfAllocate(t *testing.T) { - t.Parallel() - ctx := context.Background() - gs := framework.DefaultGameServer(framework.Namespace) - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - if err != nil { - t.Fatalf("Could not get a GameServer ready: %v", err) - } - defer framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck - - assert.Equal(t, readyGs.Status.State, agonesv1.GameServerStateReady) - reply, err := framework.SendGameServerUDP(t, readyGs, "ALLOCATE") - if err != nil { - t.Fatalf("Could not message GameServer: %v", err) - } - - assert.Equal(t, "ACK: ALLOCATE\n", reply) - gs, err = framework.WaitForGameServerState(t, readyGs, agonesv1.GameServerStateAllocated, time.Minute) - if assert.NoError(t, err) { - assert.Equal(t, agonesv1.GameServerStateAllocated, gs.Status.State) - } -} - -func TestGameServerReadyAllocateReady(t *testing.T) { - t.Parallel() - ctx := context.Background() - logger := e2eframework.TestLogger(t) - - gs := framework.DefaultGameServer(framework.Namespace) - - logger.Info("Moving to Ready") - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - require.NoError(t, err, "Could not get a GameServer ready") - logger = logger.WithField("gs", readyGs.ObjectMeta.Name) - - defer framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck - - require.Equal(t, readyGs.Status.State, agonesv1.GameServerStateReady) - - logger.Info("Moving to Allocated") - reply, err := framework.SendGameServerUDP(t, readyGs, "ALLOCATE") - require.NoError(t, err, "Could not message GameServer") - - require.Equal(t, "ACK: ALLOCATE\n", reply) - gs, err = framework.WaitForGameServerState(t, readyGs, agonesv1.GameServerStateAllocated, time.Minute) - require.NoError(t, err) - require.Equal(t, agonesv1.GameServerStateAllocated, gs.Status.State) - - logger.Info("Moving to Ready again") - reply, err = framework.SendGameServerUDP(t, readyGs, "READY") - require.NoError(t, err, "Could not message GameServer") - require.Equal(t, "ACK: READY\n", reply) - gs, err = framework.WaitForGameServerState(t, gs, agonesv1.GameServerStateReady, time.Minute) - require.NoError(t, err) - require.Equal(t, agonesv1.GameServerStateReady, gs.Status.State) -} - -func TestGameServerWithPortsMappedToMultipleContainers(t *testing.T) { - t.Parallel() - ctx := context.Background() - - firstContainerName := "udp-server" - secondContainerName := "second-udp-server" - firstPort := "gameport" - secondPort := "second-gameport" - gs := &agonesv1.GameServer{ - ObjectMeta: metav1.ObjectMeta{GenerateName: "udp-server", Namespace: framework.Namespace}, - Spec: agonesv1.GameServerSpec{ - Container: firstContainerName, - Ports: []agonesv1.GameServerPort{{ - ContainerPort: 7654, - Name: firstPort, - PortPolicy: agonesv1.Dynamic, - Protocol: corev1.ProtocolUDP, - }, { - ContainerPort: 5000, - Name: secondPort, - PortPolicy: agonesv1.Dynamic, - Protocol: corev1.ProtocolUDP, - Container: &secondContainerName, - }}, - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: firstContainerName, - Image: framework.GameServerImage, - ImagePullPolicy: corev1.PullIfNotPresent, - }, - { - Name: secondContainerName, - Image: framework.GameServerImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{"-port", "5000"}, - }, - }, - }, - }, - }, - } - - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - if err != nil { - t.Fatalf("Could not get a GameServer ready: %v", err) - } - defer framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck - assert.Equal(t, readyGs.Status.State, agonesv1.GameServerStateReady) - - interval := 2 * time.Second - timeOut := 60 * time.Second - - expectedMsg1 := "Ping 1" - errPoll := wait.PollUntilContextTimeout(context.Background(), interval, timeOut, true, func(_ context.Context) (done bool, err error) { - res, err := framework.SendGameServerUDPToPort(t, readyGs, firstPort, expectedMsg1) - if err != nil { - t.Logf("Could not message GameServer on %s: %v. Will try again...", firstPort, err) - } - return err == nil && strings.Contains(res, expectedMsg1), nil - }) - if errPoll != nil { - assert.FailNow(t, errPoll.Error(), "expected no errors after polling a port: %s", firstPort) - } - - expectedMsg2 := "Ping 2" - errPoll = wait.PollUntilContextTimeout(context.Background(), interval, timeOut, true, func(_ context.Context) (done bool, err error) { - res, err := framework.SendGameServerUDPToPort(t, readyGs, secondPort, expectedMsg2) - if err != nil { - t.Logf("Could not message GameServer on %s: %v. Will try again...", secondPort, err) - } - return err == nil && strings.Contains(res, expectedMsg2), nil - }) - - assert.NoError(t, errPoll, "expected no errors after polling a port: %s", secondPort) -} - -func TestGameServerWithPortsMappedToInitSidecarContainers(t *testing.T) { - t.Skip("Skip test. Only works on some CNIs. The current e2e platform does not support ports on init containers.") - - t.Parallel() - ctx := context.Background() - - containerName := "udp-server" - sidecarContainerName := "sidecar-server" - firstPort := "gameport" - secondPort := "second-gameport" - gs := &agonesv1.GameServer{ - ObjectMeta: metav1.ObjectMeta{GenerateName: "udp-server", Namespace: framework.Namespace}, - Spec: agonesv1.GameServerSpec{ - Container: containerName, - Ports: []agonesv1.GameServerPort{{ - ContainerPort: 7654, - Name: firstPort, - PortPolicy: agonesv1.Dynamic, - Protocol: corev1.ProtocolUDP, - }, { - ContainerPort: 5000, - Name: secondPort, - PortPolicy: agonesv1.Dynamic, - Protocol: corev1.ProtocolUDP, - Container: &sidecarContainerName, - }}, - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: containerName, - Image: framework.GameServerImage, - ImagePullPolicy: corev1.PullIfNotPresent, - }, - }, - InitContainers: []corev1.Container{ - { - Name: sidecarContainerName, - Image: framework.GameServerImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{"-port", "5000"}, - RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways), - }, - }, - }, - }, - }, - } - - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - if err != nil { - t.Fatalf("Could not get a GameServer ready: %v", err) - } - defer framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck - assert.Equal(t, readyGs.Status.State, agonesv1.GameServerStateReady) - - interval := 2 * time.Second - timeOut := 60 * time.Second - - expectedMsg1 := "Ping 1" - assert.EventuallyWithT(t, func(c *assert.CollectT) { - res, err := framework.SendGameServerUDPToPort(t, readyGs, firstPort, expectedMsg1) - require.NoError(c, err, "expected no errors after messaging GameServer a port: %s", firstPort) - assert.Contains(c, res, expectedMsg1) - }, timeOut, interval) - - expectedMsg2 := "Ping 2" - assert.EventuallyWithT(t, func(c *assert.CollectT) { - res, err := framework.SendGameServerUDPToPort(t, readyGs, secondPort, expectedMsg2) - require.NoError(c, err, "expected no errors after messaging GameServer a port: %s", secondPort) - assert.Contains(c, res, expectedMsg2) - }, timeOut, interval) -} - -func TestGameServerReserve(t *testing.T) { - t.Parallel() - ctx := context.Background() - - // We are deliberately not trying to test the transition between Reserved -> Ready. - // - // We have found that trying to catch the GameServer in the Reserved state can be flaky, - // as we can't control the speed in which the Kubernetes API is going to reply to request, - // and we could sometimes miss when the GameServer is in the Reserved State before it goes to Ready. - // - // Therefore we are going to test for concrete states that we don't need to catch while - // in a transitive state. - - gs := framework.DefaultGameServer(framework.Namespace) - gs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - if err != nil { - assert.FailNow(t, "Could not get a GameServer ready", err.Error()) - } - defer framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, gs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck - assert.Equal(t, gs.Status.State, agonesv1.GameServerStateReady) - - reply, err := framework.SendGameServerUDP(t, gs, "RESERVE 0") - if err != nil { - assert.FailNow(t, "Could not message GameServer", err.Error()) - } - assert.Equal(t, "ACK: RESERVE 0\n", reply) - - gs, err = framework.WaitForGameServerState(t, gs, agonesv1.GameServerStateReserved, 3*time.Minute) - if err != nil { - assert.FailNow(t, "Time out on waiting for gs in a Reserved state", err.Error()) - } - - reply, err = framework.SendGameServerUDP(t, gs, "ALLOCATE") - if err != nil { - assert.FailNow(t, "Could not message GameServer", err.Error()) - } - assert.Equal(t, "ACK: ALLOCATE\n", reply) - - // put it in a totally different state, just to reset things. - gs, err = framework.WaitForGameServerState(t, gs, agonesv1.GameServerStateAllocated, 3*time.Minute) - if err != nil { - assert.FailNow(t, "Time out on waiting for gs in an Allocated state", err.Error()) - } - - reply, err = framework.SendGameServerUDP(t, gs, "RESERVE 5s") - if err != nil { - assert.FailNow(t, "Could not message GameServer", err.Error()) - } - assert.Equal(t, "ACK: RESERVE 5s\n", reply) - - // sleep, since we're going to wait for the Ready response. - time.Sleep(5 * time.Second) - _, err = framework.WaitForGameServerState(t, gs, agonesv1.GameServerStateReady, 3*time.Minute) - assert.NoError(t, err) -} - -func TestGameServerShutdown(t *testing.T) { - t.Parallel() - ctx := context.Background() - gs := framework.DefaultGameServer(framework.Namespace) - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - if err != nil { - t.Fatalf("Could not get a GameServer ready: %v", err) - } - assert.Equal(t, readyGs.Status.State, agonesv1.GameServerStateReady) - - reply, err := framework.SendGameServerUDP(t, readyGs, "EXIT") - if err != nil { - t.Fatalf("Could not message GameServer: %v", err) - } - - assert.Equal(t, "ACK: EXIT\n", reply) - - err = wait.PollUntilContextTimeout(ctx, time.Second, 3*time.Minute, true, func(ctx context.Context) (bool, error) { - gs, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) - - if k8serrors.IsNotFound(err) { - return true, nil - } - - return false, err - }) - - assert.NoError(t, err) -} - -// TestGameServerEvicted test that if Gameserver would be evicted than it becomes Unhealthy -// Ephemeral Storage limit set to 0Mi -func TestGameServerEvicted(t *testing.T) { - t.Parallel() - log := e2eframework.TestLogger(t) - - ctx := context.Background() - gs := framework.DefaultGameServer(framework.Namespace) - newGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - require.NoError(t, err) - log.WithField("name", newGs.ObjectMeta.Name).Info("GameServer created, waiting for being Evicted and Unhealthy") - defer framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, newGs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck - - pods := framework.KubeClient.CoreV1().Pods(framework.Namespace) - pod, err := pods.Get(ctx, newGs.ObjectMeta.Name, metav1.GetOptions{}) - require.NoError(t, err) - - eviction := &policyv1.Eviction{ - ObjectMeta: metav1.ObjectMeta{ - Name: pod.Name, - Namespace: pod.Namespace, - }, - } - go func() { - time.Sleep(3 * time.Second) // just make sure it comes in later - log.WithField("name", eviction.ObjectMeta.Name).Info("Evicting pod!") - err := pods.EvictV1(context.Background(), eviction) - require.NoError(t, err) - }() - - _, err = framework.WaitForGameServerState(t, newGs, agonesv1.GameServerStateUnhealthy, 10*time.Minute) - require.NoError(t, err, fmt.Sprintf("waiting for [%v] GameServer Unhealthy state timed out (%v)", gs.Status.State, gs.Name)) -} - -func TestGameServerPassthroughPort(t *testing.T) { - framework.SkipOnCloudProduct(t, "gke-autopilot", "does not support Passthrough PortPolicy") - t.Parallel() - gs := framework.DefaultGameServer(framework.Namespace) - gs.Spec.Ports[0] = agonesv1.GameServerPort{PortPolicy: agonesv1.Passthrough} - gs.Spec.Template.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "PASSTHROUGH", Value: "TRUE"}} - // gate - errs := gs.Validate(agtesting.FakeAPIHooks{}) - assert.Len(t, errs, 0) - - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - if err != nil { - assert.FailNow(t, "Could not get a GameServer ready", err.Error()) - } - - port := readyGs.Spec.Ports[0] - assert.Equal(t, agonesv1.Passthrough, port.PortPolicy) - assert.NotEmpty(t, port.HostPort) - assert.Equal(t, port.HostPort, port.ContainerPort) - - reply, err := framework.SendGameServerUDP(t, readyGs, "Hello World !") - if err != nil { - t.Fatalf("Could ping GameServer: %v", err) - } - - assert.Equal(t, "ACK: Hello World !\n", reply) -} - -func TestGameServerPortPolicyNone(t *testing.T) { - if !runtime.FeatureEnabled(runtime.FeaturePortPolicyNone) { - t.SkipNow() - } - - t.Parallel() - - gs := framework.DefaultGameServer(framework.Namespace) - gs.Spec.Ports[0] = agonesv1.GameServerPort{PortPolicy: agonesv1.None, ContainerPort: 7777} - // gate - errs := gs.Validate(agtesting.FakeAPIHooks{}) - assert.Len(t, errs, 0) - - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - if err != nil { - assert.FailNow(t, "Could not get a GameServer ready", err.Error()) - } - - // To test sending UDP traffic directly to a pod when no hostPort is set, a product like Calico which uses BGP is needed - // so this only tests that the port is set correctly. - port := readyGs.Spec.Ports[0] - assert.Equal(t, agonesv1.None, port.PortPolicy) - assert.Empty(t, port.HostPort) - assert.EqualValues(t, 7777, port.ContainerPort) -} - -func TestGameServerTcpProtocol(t *testing.T) { - t.Parallel() - log := e2eframework.TestLogger(t) - ctx := context.Background() - gs := framework.DefaultGameServer(framework.Namespace) - - gs.Spec.Ports[0].Protocol = corev1.ProtocolTCP - gs.Spec.Template.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "TCP", Value: "TRUE"}} - - errs := gs.Validate(agtesting.FakeAPIHooks{}) - require.Len(t, errs, 0) - - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - require.NoError(t, err) - - replyTCP, err := e2eframework.SendGameServerTCP(readyGs, "Hello World !") - if err != nil { - framework.LogEvents(t, log, readyGs.ObjectMeta.Namespace, readyGs) - pod, err := framework.KubeClient.CoreV1().Pods(readyGs.ObjectMeta.Namespace).Get(ctx, readyGs.Name, metav1.GetOptions{}) - if err != nil { - log.WithError(err).Info("Could not retrieve pod for GameServer") - } else { - framework.LogEvents(t, log, readyGs.ObjectMeta.Namespace, pod) - framework.LogPodContainers(t, pod) - } - } - require.NoError(t, err) - assert.Equal(t, "ACK TCP: Hello World !\n", replyTCP) -} - -func TestGameServerTcpUdpProtocol(t *testing.T) { - t.Parallel() - gs := framework.DefaultGameServer(framework.Namespace) - - gs.Spec.Ports[0].Protocol = agonesv1.ProtocolTCPUDP - gs.Spec.Ports[0].Name = "gameserver" - gs.Spec.Template.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "TCP", Value: "TRUE"}} - - errs := gs.Validate(agtesting.FakeAPIHooks{}) - require.Len(t, errs, 0) - - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - if err != nil { - assert.FailNow(t, "Could not get a GameServer ready", err.Error()) - } - - tcpPort := readyGs.Spec.Ports[0] - assert.Equal(t, corev1.ProtocolTCP, tcpPort.Protocol) - assert.NotEmpty(t, tcpPort.HostPort) - assert.Equal(t, "gameserver-tcp", tcpPort.Name) - - udpPort := readyGs.Spec.Ports[1] - assert.Equal(t, corev1.ProtocolUDP, udpPort.Protocol) - assert.NotEmpty(t, udpPort.HostPort) - assert.Equal(t, "gameserver-udp", udpPort.Name) - - assert.Equal(t, tcpPort.HostPort, udpPort.HostPort) - - logrus.WithField("name", readyGs.ObjectMeta.Name).Info("GameServer created, sending UDP ping") - - replyUDP, err := framework.SendGameServerUDPToPort(t, readyGs, udpPort.Name, "Hello World !") - require.NoError(t, err) - if err != nil { - t.Fatalf("Could not ping UDP GameServer: %v", err) - } - - assert.Equal(t, "ACK: Hello World !\n", replyUDP) - - logrus.WithField("name", readyGs.ObjectMeta.Name).Info("UDP ping passed, sending TCP ping") - - replyTCP, err := e2eframework.SendGameServerTCPToPort(readyGs, tcpPort.Name, "Hello World !") - if err != nil { - t.Fatalf("Could not ping TCP GameServer: %v", err) - } - - assert.Equal(t, "ACK TCP: Hello World !\n", replyTCP) -} - -// TestGameServerStaticTcpUdpProtocol checks UDP and TCP connection for TCPUDP Protocol of Static Portpolicy -func TestGameServerStaticTcpUdpProtocol(t *testing.T) { - framework.SkipOnCloudProduct(t, "gke-autopilot", "does not support Static PortPolicy") - t.Parallel() - gs := framework.DefaultGameServer(framework.Namespace) - gs.Spec.Ports[0].PortPolicy = agonesv1.Static - gs.Spec.Ports[0].Protocol = agonesv1.ProtocolTCPUDP - gs.Spec.Ports[0].HostPort = 7000 - gs.Spec.Ports[0].Name = "gameserver" - gs.Spec.Template.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "TCP", Value: "TRUE"}} - - errs := gs.Validate(agtesting.FakeAPIHooks{}) - require.Len(t, errs, 0) - - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - require.NoError(t, err) - - tcpPort := readyGs.Spec.Ports[0] - assert.Equal(t, corev1.ProtocolTCP, tcpPort.Protocol) - assert.NotEmpty(t, tcpPort.HostPort) - assert.Equal(t, "gameserver-tcp", tcpPort.Name) - - udpPort := readyGs.Spec.Ports[1] - assert.Equal(t, corev1.ProtocolUDP, udpPort.Protocol) - assert.NotEmpty(t, udpPort.HostPort) - assert.Equal(t, "gameserver-udp", udpPort.Name) - - assert.Equal(t, tcpPort.HostPort, udpPort.HostPort) - - logrus.WithField("name", readyGs.ObjectMeta.Name).Info("GameServer created, sending UDP ping") - - replyUDP, err := framework.SendGameServerUDPToPort(t, readyGs, udpPort.Name, "Hello World !") - require.NoError(t, err) - if err != nil { - t.Fatalf("Could not ping UDP GameServer: %v", err) - } - - assert.Equal(t, "ACK: Hello World !\n", replyUDP) - - logrus.WithField("name", readyGs.ObjectMeta.Name).Info("UDP ping passed, sending TCP ping") - - replyTCP, err := e2eframework.SendGameServerTCPToPort(readyGs, tcpPort.Name, "Hello World !") - if err != nil { - t.Fatalf("Could not ping TCP GameServer: %v", err) - } - - assert.Equal(t, "ACK TCP: Hello World !\n", replyTCP) -} - -// TestGameServerStaticTcpProtocol checks TCP connection for TCP Protocol of Static Portpolicy -func TestGameServerStaticTcpProtocol(t *testing.T) { - framework.SkipOnCloudProduct(t, "gke-autopilot", "does not support Static PortPolicy") - t.Parallel() - gs := framework.DefaultGameServer(framework.Namespace) - - gs.Spec.Ports[0].PortPolicy = agonesv1.Static - gs.Spec.Ports[0].Protocol = corev1.ProtocolTCP - gs.Spec.Ports[0].HostPort = 7000 - gs.Spec.Template.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "TCP", Value: "TRUE"}} - - errs := gs.Validate(agtesting.FakeAPIHooks{}) - require.Len(t, errs, 0) - - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - require.NoError(t, err) - - logrus.WithField("name", readyGs.ObjectMeta.Name).Info("sending TCP ping") - - replyTCP, err := e2eframework.SendGameServerTCP(readyGs, "Hello World !") - require.NoError(t, err) - assert.Equal(t, "ACK TCP: Hello World !\n", replyTCP) - - logrus.WithField("name", readyGs.ObjectMeta.Name).Info("TCP ping Passed") -} - -// TestGameServerStaticUdpProtocol checks default(UDP) connection of Static Portpolicy -func TestGameServerStaticUdpProtocol(t *testing.T) { - framework.SkipOnCloudProduct(t, "gke-autopilot", "does not support Static PortPolicy") - t.Parallel() - gs := framework.DefaultGameServer(framework.Namespace) - - gs.Spec.Ports[0].PortPolicy = agonesv1.Static - gs.Spec.Ports[0].HostPort = 7000 - - errs := gs.Validate(agtesting.FakeAPIHooks{}) - require.Len(t, errs, 0) - - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - require.NoError(t, err) - - logrus.WithField("name", readyGs.ObjectMeta.Name).Info("sending UDP ping") - - replyTCP, err := framework.SendGameServerUDP(t, readyGs, "Default UDP connection check") - require.NoError(t, err) - assert.Equal(t, "ACK: Default UDP connection check\n", replyTCP) - - logrus.WithField("name", readyGs.ObjectMeta.Name).Info("UDP ping Passed") -} - -func TestGameServerWithoutPort(t *testing.T) { - t.Parallel() - gs := framework.DefaultGameServer(framework.Namespace) - gs.Spec.Ports = nil - - errs := gs.Validate(agtesting.FakeAPIHooks{}) - assert.Len(t, errs, 0) - - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - - require.NoError(t, err, "Could not get a GameServer ready") - assert.Empty(t, readyGs.Spec.Ports) -} - -// TestGameServerResourceValidation - check that we are not able to use -// invalid PodTemplate for GameServer Spec with wrong Resource Requests and Limits -func TestGameServerResourceValidation(t *testing.T) { - t.Parallel() - ctx := context.Background() - gs := framework.DefaultGameServer(framework.Namespace) - mi64 := resource.MustParse("64Mi") - gs.Spec.Template.Spec.Containers[0].Resources.Limits[corev1.ResourceMemory] = mi64 - gs.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceMemory] = resource.MustParse("128Mi") - - errs := gs.Validate(agtesting.FakeAPIHooks{}) - assert.False(t, len(errs) == 0) - - gsClient := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace) - - _, err := gsClient.Create(ctx, gs.DeepCopy(), metav1.CreateOptions{}) - assert.NotNil(t, err) - statusErr, ok := err.(*k8serrors.StatusError) - assert.True(t, ok) - assert.Len(t, statusErr.Status().Details.Causes, 1) - assert.Equal(t, metav1.CauseTypeFieldValueInvalid, statusErr.Status().Details.Causes[0].Type) - assert.Equal(t, "spec.template.spec.containers[0].resources.requests", statusErr.Status().Details.Causes[0].Field) - - gs.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = resource.MustParse("-50m") - _, err = gsClient.Create(ctx, gs.DeepCopy(), metav1.CreateOptions{}) - assert.NotNil(t, err) - statusErr, ok = err.(*k8serrors.StatusError) - assert.True(t, ok) - assert.Len(t, statusErr.Status().Details.Causes, 2) - sort.Slice(statusErr.Status().Details.Causes, func(i, j int) bool { - return statusErr.Status().Details.Causes[i].Field > statusErr.Status().Details.Causes[j].Field - }) - assert.Equal(t, metav1.CauseTypeFieldValueInvalid, statusErr.Status().Details.Causes[0].Type) - assert.Equal(t, "spec.template.spec.containers[0].resources.requests[cpu]", statusErr.Status().Details.Causes[0].Field) - - // test that values are still being set correctly - m50 := resource.MustParse("50m") - gs.Spec.Template.Spec.Containers[0].Resources.Limits[corev1.ResourceMemory] = mi64 - gs.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceMemory] = mi64 - gs.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = m50 - gs.Spec.Template.Spec.Containers[0].Resources.Limits[corev1.ResourceCPU] = m50 - - // confirm we have a valid GameServer before running the test - errs = gs.Validate(agtesting.FakeAPIHooks{}) - require.Len(t, errs, 0) - - gsCopy, err := gsClient.Create(ctx, gs.DeepCopy(), metav1.CreateOptions{}) - require.NoError(t, err) - assert.Equal(t, mi64, gsCopy.Spec.Template.Spec.Containers[0].Resources.Limits[corev1.ResourceMemory]) - assert.Equal(t, mi64, gsCopy.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceMemory]) - assert.Equal(t, m50, gsCopy.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU]) - assert.Equal(t, m50, gsCopy.Spec.Template.Spec.Containers[0].Resources.Limits[corev1.ResourceCPU]) -} - -func TestGameServerPodTemplateSpecFailSchemaValidation(t *testing.T) { - t.Parallel() - - // The Kubernetes dynamic client skips schema validation (for reasons I'm not sure of), so the - // best way I could determine to test schema validation is via calling kubectl directly. - // The schema's from Kubernetes don't include anything like `pattern:` or `enum:` which would - // potentially make this easier to test. - - gsYaml := ` -apiVersion: "agones.dev/v1" -kind: GameServer -metadata: - name: "invalid-gameserver" -spec: - ports: - - name: default - portPolicy: Dynamic - containerPort: 7654 - template: - spec: - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: ERROR - containers: - - name: simple-game-server - image: us-docker.pkg.dev/agones-images/examples/simple-game-server:0.43 -` - err := os.WriteFile("/tmp/invalid.yaml", []byte(gsYaml), 0o644) - require.NoError(t, err) - - cmd := exec.Command("kubectl", "apply", "-f", "/tmp/invalid.yaml") - var stdout bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - err = cmd.Run() - logrus.WithField("stdout", stdout.String()).WithField("stderr", stderr.String()).WithError(err).Info("Ran command!") - require.Error(t, err) - assert.Contains(t, stderr.String(), "spec.template.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution") -} - -func TestGameServerSetPlayerCapacity(t *testing.T) { - if !runtime.FeatureEnabled(runtime.FeaturePlayerTracking) { - t.SkipNow() - } - t.Parallel() - ctx := context.Background() - - t.Run("no initial capacity set", func(t *testing.T) { - gs := framework.DefaultGameServer(framework.Namespace) - gs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - if err != nil { - t.Fatalf("Could not get a GameServer ready: %v", err) - } - assert.Equal(t, gs.Status.State, agonesv1.GameServerStateReady) - assert.Equal(t, int64(0), gs.Status.Players.Capacity) - - reply, err := framework.SendGameServerUDP(t, gs, "PLAYER_CAPACITY") - if err != nil { - t.Fatalf("Could not message GameServer: %v", err) - } - assert.Equal(t, "0\n", reply) - - reply, err = framework.SendGameServerUDP(t, gs, "PLAYER_CAPACITY 20") - if err != nil { - t.Fatalf("Could not message GameServer: %v", err) - } - assert.Equal(t, "ACK: PLAYER_CAPACITY 20\n", reply) - - reply, err = framework.SendGameServerUDP(t, gs, "PLAYER_CAPACITY") - if err != nil { - t.Fatalf("Could not message GameServer: %v", err) - } - assert.Equal(t, "20\n", reply) - - err = wait.PollUntilContextTimeout(ctx, time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { - gs, err := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, gs.ObjectMeta.Name, metav1.GetOptions{}) - if err != nil { - return false, err - } - return gs.Status.Players.Capacity == 20, nil - }) - assert.NoError(t, err) - }) - - t.Run("initial capacity set", func(t *testing.T) { - gs := framework.DefaultGameServer(framework.Namespace) - gs.Spec.Players = &agonesv1.PlayersSpec{InitialCapacity: 10} - gs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - if err != nil { - t.Fatalf("Could not get a GameServer ready: %v", err) - } - assert.Equal(t, gs.Status.State, agonesv1.GameServerStateReady) - assert.Equal(t, int64(10), gs.Status.Players.Capacity) - - reply, err := framework.SendGameServerUDP(t, gs, "PLAYER_CAPACITY") - if err != nil { - t.Fatalf("Could not message GameServer: %v", err) - } - assert.Equal(t, "10\n", reply) - - reply, err = framework.SendGameServerUDP(t, gs, "PLAYER_CAPACITY 20") - if err != nil { - t.Fatalf("Could not message GameServer: %v", err) - } - assert.Equal(t, "ACK: PLAYER_CAPACITY 20\n", reply) - - reply, err = framework.SendGameServerUDP(t, gs, "PLAYER_CAPACITY") - if err != nil { - t.Fatalf("Could not message GameServer: %v", err) - } - assert.Equal(t, "20\n", reply) - - err = wait.PollUntilContextTimeout(context.Background(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { - gs, err := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, gs.ObjectMeta.Name, metav1.GetOptions{}) - if err != nil { - return false, err - } - return gs.Status.Players.Capacity == 20, nil - }) - assert.NoError(t, err) - - time.Sleep(30 * time.Second) - }) -} - -func TestPlayerConnectWithCapacityZero(t *testing.T) { - if !runtime.FeatureEnabled(runtime.FeaturePlayerTracking) { - t.SkipNow() - } - t.Parallel() - - gs := framework.DefaultGameServer(framework.Namespace) - playerCount := int64(0) - gs.Spec.Players = &agonesv1.PlayersSpec{InitialCapacity: playerCount} - gs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - require.NoError(t, err) - assert.Equal(t, gs.Status.State, agonesv1.GameServerStateReady) - assert.Equal(t, playerCount, gs.Status.Players.Capacity) - - // add a player - msg := "PLAYER_CONNECT 1" - logrus.WithField("msg", msg).Info("Sending Player Connect") - _, err = framework.SendGameServerUDP(t, gs, msg) - // expected error from the log.Fatalf("could not connect player: %v", err) - if assert.Error(t, err) { - _, err := framework.WaitForGameServerState(t, gs, agonesv1.GameServerStateUnhealthy, time.Minute) - assert.NoError(t, err) - } -} - -func TestPlayerConnectAndDisconnect(t *testing.T) { - if !runtime.FeatureEnabled(runtime.FeaturePlayerTracking) { - t.SkipNow() - } - t.Parallel() - ctx := context.Background() - - gs := framework.DefaultGameServer(framework.Namespace) - playerCount := int64(3) - gs.Spec.Players = &agonesv1.PlayersSpec{InitialCapacity: playerCount} - gs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - if err != nil { - t.Fatalf("Could not get a GameServer ready: %v", err) - } - assert.Equal(t, gs.Status.State, agonesv1.GameServerStateReady) - assert.Equal(t, playerCount, gs.Status.Players.Capacity) - - // add three players in quick succession - for i := int64(1); i <= playerCount; i++ { - msg := "PLAYER_CONNECT " + fmt.Sprintf("%d", i) - logrus.WithField("msg", msg).Info("Sending Player Connect") - reply, err := framework.SendGameServerUDP(t, gs, msg) - if err != nil { - t.Fatalf("Could not message GameServer: %v", err) - } - assert.Equal(t, fmt.Sprintf("ACK: %s\n", msg), reply) - } - - // deliberately do this before polling, to test the SDK returning the correct - // results before it is committed to the GameServer resource. - reply, err := framework.SendGameServerUDP(t, gs, "PLAYER_CONNECTED 1") - if err != nil { - t.Fatalf("Could not message GameServer: %v", err) - } - assert.Equal(t, "true\n", reply) - - reply, err = framework.SendGameServerUDP(t, gs, "GET_PLAYERS") - if err != nil { - t.Fatalf("Could not message GameServer: %v", err) - } - assert.ElementsMatch(t, []string{"1", "2", "3"}, strings.Split(strings.TrimSpace(reply), ",")) - - reply, err = framework.SendGameServerUDP(t, gs, "PLAYER_COUNT") - if err != nil { - t.Fatalf("Could not message GameServer: %v", err) - } - assert.Equal(t, "3\n", reply) - - err = wait.PollUntilContextTimeout(ctx, time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { - gs, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, gs.ObjectMeta.Name, metav1.GetOptions{}) - if err != nil { - return false, err - } - return gs.Status.Players.Count == playerCount, nil - }) - assert.NoError(t, err) - assert.ElementsMatch(t, []string{"1", "2", "3"}, gs.Status.Players.IDs) - - // let's disconnect player 2 - logrus.Info("Disconnect Player 2") - reply, err = framework.SendGameServerUDP(t, gs, "PLAYER_DISCONNECT 2") - if err != nil { - t.Fatalf("Could not message GameServer: %v", err) - } - assert.Equal(t, "ACK: PLAYER_DISCONNECT 2\n", reply) - - reply, err = framework.SendGameServerUDP(t, gs, "PLAYER_CONNECTED 2") - if err != nil { - t.Fatalf("Could not message GameServer: %v", err) - } - assert.Equal(t, "false\n", reply) - - reply, err = framework.SendGameServerUDP(t, gs, "GET_PLAYERS") - if err != nil { - t.Fatalf("Could not message GameServer: %v", err) - } - assert.ElementsMatch(t, []string{"1", "3"}, strings.Split(strings.TrimSpace(reply), ",")) - - reply, err = framework.SendGameServerUDP(t, gs, "PLAYER_COUNT") - if err != nil { - t.Fatalf("Could not message GameServer: %v", err) - } - assert.Equal(t, "2\n", reply) - - err = wait.PollUntilContextTimeout(ctx, time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { - gs, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, gs.ObjectMeta.Name, metav1.GetOptions{}) - if err != nil { - return false, err - } - return gs.Status.Players.Count == 2, nil - }) - assert.NoError(t, err) - assert.ElementsMatch(t, []string{"1", "3"}, gs.Status.Players.IDs) -} - -func TestCounters(t *testing.T) { - if !runtime.FeatureEnabled(runtime.FeatureCountsAndLists) { - t.SkipNow() - } - t.Parallel() - ctx := context.Background() - gs := framework.DefaultGameServer(framework.Namespace) - - gs.Spec.Counters = make(map[string]agonesv1.CounterStatus) - gs.Spec.Counters["games"] = agonesv1.CounterStatus{ - Count: 1, - Capacity: 50, - } - gs.Spec.Counters["foo"] = agonesv1.CounterStatus{ - Count: 10, - Capacity: 100, - } - gs.Spec.Counters["bar"] = agonesv1.CounterStatus{ - Count: 10, - Capacity: 10, - } - gs.Spec.Counters["baz"] = agonesv1.CounterStatus{ - Count: 1000, - Capacity: 1000, - } - gs.Spec.Counters["qux"] = agonesv1.CounterStatus{ - Count: 42, - Capacity: 50, - } - - gs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - require.NoError(t, err) - defer framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, gs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck - assert.Equal(t, agonesv1.GameServerStateReady, gs.Status.State) - - testCases := map[string]struct { - msg string - want string - counterName string - wantCount string - wantCapacity string - }{ - "GetCounterCount": { - msg: "GET_COUNTER_COUNT games", - want: "COUNTER: 1\n", - }, - "GetCounterCount Counter Does Not Exist": { - msg: "GET_COUNTER_COUNT fame", - want: "ERROR: -1\n", - }, - "IncrementCounter": { - msg: "INCREMENT_COUNTER foo 10", - want: "SUCCESS\n", - counterName: "foo", - wantCount: "COUNTER: 20\n", - }, - "IncrementCounter Past Capacity": { - msg: "INCREMENT_COUNTER games 50", - want: "ERROR: could not increment Counter games by amount 50: rpc error: code = Unknown desc = out of range. Count must be within range [0,Capacity]. Found Count: 51, Capacity: 50\n", - counterName: "games", - wantCount: "COUNTER: 1\n", - }, - "IncrementCounter Negative": { - msg: "INCREMENT_COUNTER games -1", - want: "ERROR: amount must be a positive int64, found -1\n", - counterName: "games", - wantCount: "COUNTER: 1\n", - }, - "IncrementCounter Counter Does Not Exist": { - msg: "INCREMENT_COUNTER same 1", - want: "ERROR: could not increment Counter same by amount 1: rpc error: code = Unknown desc = counter not found: same\n", - }, - "DecrementCounter": { - msg: "DECREMENT_COUNTER bar 10", - want: "SUCCESS\n", - counterName: "bar", - wantCount: "COUNTER: 0\n", - }, - "DecrementCounter Past Capacity": { - msg: "DECREMENT_COUNTER games 2", - want: "ERROR: could not decrement Counter games by amount 2: rpc error: code = Unknown desc = out of range. Count must be within range [0,Capacity]. Found Count: -1, Capacity: 50\n", - counterName: "games", - wantCount: "COUNTER: 1\n", - }, - "DecrementCounter Negative": { - msg: "DECREMENT_COUNTER games -1", - want: "ERROR: amount must be a positive int64, found -1\n", - counterName: "games", - wantCount: "COUNTER: 1\n", - }, - "DecrementCounter Counter Does Not Exist": { - msg: "DECREMENT_COUNTER lame 1", - want: "ERROR: could not decrement Counter lame by amount 1: rpc error: code = Unknown desc = counter not found: lame\n", - }, - "SetCounterCount": { - msg: "SET_COUNTER_COUNT baz 0", - want: "SUCCESS\n", - counterName: "baz", - wantCount: "COUNTER: 0\n", - }, - "SetCounterCount Past Capacity": { - msg: "SET_COUNTER_COUNT games 51", - want: "ERROR: could not set Counter games count to amount 51: rpc error: code = Unknown desc = out of range. Count must be within range [0,Capacity]. Found Count: 51, Capacity: 50\n", - counterName: "games", - wantCount: "COUNTER: 1\n", - }, - "SetCounterCount Past Zero": { - msg: "SET_COUNTER_COUNT games -1", - want: "ERROR: could not set Counter games count to amount -1: rpc error: code = Unknown desc = out of range. Count must be within range [0,Capacity]. Found Count: -1, Capacity: 50\n", - counterName: "games", - wantCount: "COUNTER: 1\n", - }, - "GetCounterCapacity": { - msg: "GET_COUNTER_CAPACITY games", - want: "CAPACITY: 50\n", - }, - "GetCounterCapacity Counter Does Not Exist": { - msg: "GET_COUNTER_CAPACITY dame", - want: "ERROR: -1\n", - }, - "SetCounterCapacity": { - msg: "SET_COUNTER_CAPACITY qux 0", - want: "SUCCESS\n", - counterName: "qux", - wantCapacity: "CAPACITY: 0\n", - }, - "SetCounterCapacity Past Zero": { - msg: "SET_COUNTER_CAPACITY games -42", - want: "ERROR: could not set Counter games capacity to amount -42: rpc error: code = Unknown desc = out of range. Capacity must be greater than or equal to 0. Found Capacity: -42\n", - counterName: "games", - wantCount: "COUNTER: 1\n", - }, - } - // nolint:dupl // Linter errors on lines are duplicate of TestLists - for name, testCase := range testCases { - t.Run(name, func(t *testing.T) { - logrus.WithField("msg", testCase.msg).Info(name) - reply, err := framework.SendGameServerUDP(t, gs, testCase.msg) - require.NoError(t, err) - assert.Equal(t, testCase.want, reply) - - if testCase.wantCount != "" { - msg := "GET_COUNTER_COUNT " + testCase.counterName - logrus.WithField("msg", msg).Info("Sending GetCounterCount") - reply, err = framework.SendGameServerUDP(t, gs, msg) - require.NoError(t, err) - assert.Equal(t, testCase.wantCount, reply) - } - - if testCase.wantCapacity != "" { - msg := "GET_COUNTER_CAPACITY " + testCase.counterName - logrus.WithField("msg", msg).Info("Sending GetCounterCapacity") - reply, err = framework.SendGameServerUDP(t, gs, msg) - require.NoError(t, err) - assert.Equal(t, testCase.wantCapacity, reply) - } - }) - } -} - -func TestLists(t *testing.T) { - if !runtime.FeatureEnabled(runtime.FeatureCountsAndLists) { - t.SkipNow() - } - t.Parallel() - ctx := context.Background() - gs := framework.DefaultGameServer(framework.Namespace) - gs.Labels = map[string]string{agonesv1.FleetNameLabel: "fleet-example"} - gs.Spec.Lists = make(map[string]agonesv1.ListStatus) - gs.Spec.Lists["players"] = agonesv1.ListStatus{ - Capacity: 1000, - } - gs.Spec.Lists["games"] = agonesv1.ListStatus{ - Values: []string{"game1", "game2"}, - Capacity: 50, - } - gs.Spec.Lists["foo"] = agonesv1.ListStatus{ - Values: []string{}, - Capacity: 1, - } - gs.Spec.Lists["bar"] = agonesv1.ListStatus{ - Values: []string{"bar1", "bar2"}, - Capacity: 10, - } - gs.Spec.Lists["baz"] = agonesv1.ListStatus{ - Values: []string{"baz1"}, - Capacity: 1, - } - gs.Spec.Lists["qux"] = agonesv1.ListStatus{ - Values: []string{"qux1", "qux2", "qux3", "qux4"}, - Capacity: 5, - } - - gs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - require.NoError(t, err) - defer framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, gs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck - assert.Equal(t, agonesv1.GameServerStateReady, gs.Status.State) - - testCases := map[string]struct { - msg string - want string - listName string - wantLength string - wantCapacity string - }{ - "GetListCapacity": { - msg: "GET_LIST_CAPACITY games", - want: "CAPACITY: 50\n", - }, - "SetListCapacity": { - msg: "SET_LIST_CAPACITY foo 1000", - want: "SUCCESS\n", - listName: "foo", - wantCapacity: "CAPACITY: 1000\n", - }, - "SetListCapacity past 1000": { - msg: "SET_LIST_CAPACITY games 1001", - want: "ERROR: could not set List games capacity to amount 1001: rpc error: code = Unknown desc = out of range. Capacity must be within range [0,1000]. Found Capacity: 1001\n", - listName: "games", - wantCapacity: "CAPACITY: 50\n", - }, - "SetListCapacity negative": { - msg: "SET_LIST_CAPACITY games -1", - want: "ERROR: could not set List games capacity to amount -1: rpc error: code = Unknown desc = out of range. Capacity must be within range [0,1000]. Found Capacity: -1\n", - listName: "games", - wantCapacity: "CAPACITY: 50\n", - }, - "ListContains": { - msg: "LIST_CONTAINS games game2", - want: "FOUND: true\n", - }, - "ListContains false": { - msg: "LIST_CONTAINS games game0", - want: "FOUND: false\n", - }, - "GetListLength": { - msg: "GET_LIST_LENGTH games", - want: "LENGTH: 2\n", - }, - "GetListValues": { - msg: "GET_LIST_VALUES games", - want: "VALUES: game1,game2\n", - }, - "GetListValues empty": { - msg: "GET_LIST_VALUES foo", - want: "VALUES: \n", - }, - "AppendListValue": { - msg: "APPEND_LIST_VALUE bar bar3", - want: "SUCCESS\n", - listName: "bar", - wantLength: "LENGTH: 3\n", - }, - "AppendListValue past capacity": { - msg: "APPEND_LIST_VALUE baz baz2", - want: "ERROR: could not get List baz: rpc error: code = Unknown desc = out of range. No available capacity. Current Capacity: 1, List Size: 1\n", - listName: "baz", - wantLength: "LENGTH: 1\n", - }, - "DeleteListValue": { - msg: "DELETE_LIST_VALUE qux qux3", - want: "SUCCESS\n", - listName: "qux", - wantLength: "LENGTH: 3\n", - }, - "DeleteListValue value does not exist": { - msg: "DELETE_LIST_VALUE games game4", - want: "ERROR: could not get List games: rpc error: code = Unknown desc = not found: value game4 not in list games\n", - listName: "games", - wantLength: "LENGTH: 2\n", - }, - } - // nolint:dupl // Linter errors on lines are duplicate of TestCounters - for name, testCase := range testCases { - t.Run(name, func(t *testing.T) { - logrus.WithField("msg", testCase.msg).Info(name) - reply, err := framework.SendGameServerUDP(t, gs, testCase.msg) - require.NoError(t, err) - assert.Equal(t, testCase.want, reply) - - if testCase.wantLength != "" { - msg := "GET_LIST_LENGTH " + testCase.listName - logrus.WithField("msg", msg).Info("Sending GetListLength") - reply, err = framework.SendGameServerUDP(t, gs, msg) - require.NoError(t, err) - assert.Equal(t, testCase.wantLength, reply) - } - - if testCase.wantCapacity != "" { - msg := "GET_LIST_CAPACITY " + testCase.listName - logrus.WithField("msg", msg).Info("Sending GetListCapacity") - reply, err = framework.SendGameServerUDP(t, gs, msg) - require.NoError(t, err) - assert.Equal(t, testCase.wantCapacity, reply) - } - }) - } -} - -func TestSideCarCommunicatesWhileTerminating(t *testing.T) { - t.Parallel() - ctx := context.Background() - gs := framework.DefaultGameServer(framework.Namespace) - - minute := int64(60) - gs.Spec.Template.Spec.Containers[0].Args = append(gs.Spec.Template.Spec.Containers[0].Args, "--gracefulTerminationDelaySec", "60") - gs.Spec.Template.Spec.TerminationGracePeriodSeconds = &minute - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - require.NoError(t, err) - require.Equal(t, readyGs.Status.State, agonesv1.GameServerStateReady) - - // delete the GameServer - gameServers := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace) - err = gameServers.Delete(context.Background(), readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) - require.NoError(t, err, "Could not delete GameServer") - - // wait for the deletion timestamp to be set - err = wait.PollUntilContextTimeout(ctx, time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { - gs, err := gameServers.Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) - if err != nil { - return false, err - } - return gs.DeletionTimestamp != nil, nil - }) - require.NoError(t, err, "Could not get a GameServer with deletion timestamp") - - // send a "GAMESERVER" message, and confirm it works - reply, err := framework.SendGameServerUDP(t, readyGs, "GAMESERVER") - require.NoError(t, err) - require.Equal(t, fmt.Sprintf("NAME: %s\n", readyGs.ObjectMeta.Name), reply) -} - -func TestGracefulShutdown(t *testing.T) { - // with the new sidecar pattern, there's no need for waiting on the Shutdown state. - if runtime.FeatureEnabled(runtime.FeatureSidecarContainers) { - t.SkipNow() - } - - t.Parallel() - - log := e2eframework.TestLogger(t) - ctx := context.Background() - gs := framework.DefaultGameServer(framework.Namespace) - var minute int64 = 60 - gs.Spec.Template.Spec.TerminationGracePeriodSeconds = &minute - readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - if err != nil { - t.Fatalf("Could not get a GameServer ready: %v", err) - } - assert.Equal(t, readyGs.Status.State, agonesv1.GameServerStateReady) - gameservers := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace) - err = gameservers.Delete(ctx, readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) - require.NoError(t, err) - log.Info("Deleted GameServer, waiting 20 seconds...") - time.Sleep(20 * time.Second) - log.WithField("gs", gs).Info("Checking GameServer") - gs, err = gameservers.Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) - require.NoError(t, err) - assert.Equal(t, readyGs.ObjectMeta.Name, gs.ObjectMeta.Name) - - // move it to shutdown - gsCopy := gs.DeepCopy() - gsCopy.Status.State = agonesv1.GameServerStateShutdown - _, err = gameservers.Update(ctx, gsCopy, metav1.UpdateOptions{}) - require.NoError(t, err) - - start := time.Now() - require.Eventually(t, func() bool { - _, err := gameservers.Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) - log.WithError(err).Info("checking GameServer") - if err == nil { - return false - } - return k8serrors.IsNotFound(err) - }, 40*time.Second, time.Second) - - diff := int(time.Since(start).Seconds()) - log.WithField("diff", diff).Info("Time difference") - require.Less(t, diff, 40) -} - -func TestGameServerSlowStart(t *testing.T) { - t.Parallel() - - // Inject an additional game server sidecar that forces a delayed start - // to the main game server container following the pattern at - // https://medium.com/@marko.luksa/delaying-application-start-until-sidecar-is-ready-2ec2d21a7b74 - gs := framework.DefaultGameServer(framework.Namespace) - gs.Spec.Template.Spec.Containers = append( - []corev1.Container{{ - Name: "delay-game-server-start", - Image: "alpine:latest", - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"sleep", "3600"}, - Lifecycle: &corev1.Lifecycle{ - PostStart: &corev1.LifecycleHandler{ - Exec: &corev1.ExecAction{ - Command: []string{"sleep", "60"}, - }, - }, - }, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("30m"), - corev1.ResourceMemory: resource.MustParse("64Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("30m"), - corev1.ResourceMemory: resource.MustParse("64Mi"), - }, - }, - }}, - gs.Spec.Template.Spec.Containers...) - - // Validate that a game server whose primary container starts slowly (a full minute - // after the SDK starts) is capable of reaching Ready. Here we force the condition - // with a lifecycle hook, but it imitates a slow image pull, or other container - // start delays. - _, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - assert.NoError(t, err) -} - -func TestGameServerPatch(t *testing.T) { - if !runtime.FeatureEnabled(runtime.FeatureCountsAndLists) { - t.SkipNow() - } - t.Parallel() - ctx := context.Background() - - gs := framework.DefaultGameServer(framework.Namespace) - gs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) - require.NoError(t, err) - defer framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, gs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck - assert.Equal(t, agonesv1.GameServerStateReady, gs.Status.State) - - // Create a gameserver to patch against - gsCopy := gs.DeepCopy() - gsCopy.ObjectMeta.Labels = map[string]string{"foo": "foo-value"} - - patch, err := gs.Patch(gsCopy) - require.NoError(t, err) - patchString := string(patch) - require.Contains(t, patchString, fmt.Sprintf("{\"op\":\"test\",\"path\":\"/metadata/resourceVersion\",\"value\":%q}", gs.ObjectMeta.ResourceVersion)) - require.Contains(t, patchString, "{\"op\":\"add\",\"path\":\"/metadata/labels\",\"value\":{\"foo\":\"foo-value\"}}") - - // Confirm patch is applied correctly - patchedGs, err := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Patch(ctx, gs.GetObjectMeta().GetName(), types.JSONPatchType, patch, metav1.PatchOptions{}) - require.NoError(t, err) - require.Equal(t, patchedGs.ObjectMeta.Labels, map[string]string{"foo": "foo-value"}) - require.NotEqual(t, patchedGs.ObjectMeta.ResourceVersion, gs.ObjectMeta.ResourceVersion) - - // Confirm a patch applied to an old version of a game server is not applied - gsCopy.ObjectMeta.Labels = map[string]string{"bar": "bar-value"} - patch, err = gs.Patch(gsCopy) - require.NoError(t, err) - - _, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Patch(ctx, gs.GetObjectMeta().GetName(), types.JSONPatchType, patch, metav1.PatchOptions{}) - require.Error(t, err) - - getGs, err := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, gs.ObjectMeta.Name, metav1.GetOptions{}) - require.NoError(t, err) - require.Equal(t, getGs.ObjectMeta.Labels, map[string]string{"foo": "foo-value"}) - require.Equal(t, getGs.ObjectMeta.ResourceVersion, patchedGs.ObjectMeta.ResourceVersion) - - // Confirm patch goes through with the most up-to-date game server - gsCopy = patchedGs.DeepCopy() - gsCopy.ObjectMeta.Labels = map[string]string{"bar": "bar-value"} - patch, err = patchedGs.Patch(gsCopy) - require.NoError(t, err) - - rePatchedGs, err := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Patch(ctx, gs.GetObjectMeta().GetName(), types.JSONPatchType, patch, metav1.PatchOptions{}) - require.NoError(t, err) - require.Equal(t, rePatchedGs.ObjectMeta.Labels, map[string]string{"bar": "bar-value"}) - - getGs, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, gs.ObjectMeta.Name, metav1.GetOptions{}) - require.NoError(t, err) - require.Equal(t, getGs.ObjectMeta.Labels, map[string]string{"bar": "bar-value"}) - require.Equal(t, getGs.ObjectMeta.ResourceVersion, rePatchedGs.ObjectMeta.ResourceVersion) -} +// Copyright Contributors to Agones a Series of LF Projects, LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bytes" + "context" + "fmt" + "net" + "os" + "os/exec" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/utils/ptr" + + agonesv1 "agones.dev/agones/pkg/apis/agones/v1" + allocationv1 "agones.dev/agones/pkg/apis/allocation/v1" + agtesting "agones.dev/agones/pkg/testing" + "agones.dev/agones/pkg/util/runtime" + e2eframework "agones.dev/agones/test/e2e/framework" +) + +const ( + fakeIPAddress = "192.1.1.2" +) + +func TestCreateConnect(t *testing.T) { + t.Parallel() + ctx := context.Background() + gs := framework.DefaultGameServer(framework.Namespace) + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + if err != nil { + t.Fatalf("Could not get a GameServer ready: %v", err) + } + assert.Equal(t, len(readyGs.Status.Ports), 1) + assert.NotEmpty(t, readyGs.Status.Ports[0].Port) + assert.NotEmpty(t, readyGs.Status.Address) + assert.NotEmpty(t, readyGs.Status.Addresses) + + require.NotEmpty(t, readyGs.Status.NodeName) + require.Equal(t, readyGs.Status.State, agonesv1.GameServerStateReady) + + // check connectivity before anything else. + reply, err := framework.SendGameServerUDP(t, readyGs, "Hello World !") + require.NoError(t, err) + require.Equal(t, "ACK: Hello World !\n", reply) + + // since the PodIP could come at any point, let's eventually it. + require.EventuallyWithT(t, func(c *assert.CollectT) { + readyGs, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) + require.NoError(c, err) + for i, addr := range readyGs.Status.Addresses { + if addr.Type == agonesv1.NodePodIP { + require.NotEmpty(c, readyGs.Status.Addresses[i].Address) + } + } + }, 3*time.Minute, time.Second, "Failed to get PodIP") +} + +func TestHostName(t *testing.T) { + t.Parallel() + + pods := framework.KubeClient.CoreV1().Pods(framework.Namespace) + + fixtures := map[string]struct { + setup func(gs *agonesv1.GameServer) + test func(gs *agonesv1.GameServer, pod *corev1.Pod) + }{ + "standard hostname": { + setup: func(_ *agonesv1.GameServer) {}, + test: func(gs *agonesv1.GameServer, pod *corev1.Pod) { + assert.Equal(t, gs.ObjectMeta.Name, pod.Spec.Hostname) + }, + }, + "a . in the name": { + setup: func(gs *agonesv1.GameServer) { + gs.ObjectMeta.GenerateName = "game-server-1.0-" + }, + test: func(_ *agonesv1.GameServer, pod *corev1.Pod) { + expected := "game-server-1-0-" + // since it's a generated name, we just check the beginning. + assert.Equal(t, expected, pod.Spec.Hostname[:len(expected)]) + }, + }, + // generated name will automatically truncate to 63 chars. + "generated with > 63 chars": { + setup: func(gs *agonesv1.GameServer) { + gs.ObjectMeta.GenerateName = "game-server-" + strings.Repeat("n", 100) + }, + test: func(gs *agonesv1.GameServer, pod *corev1.Pod) { + assert.Equal(t, gs.ObjectMeta.Name, pod.Spec.Hostname) + }, + }, + // Note: no need to test for a gs.ObjectMeta.Name > 63 chars, as it will be rejected as invalid + } + + for k, v := range fixtures { + t.Run(k, func(t *testing.T) { + gs := framework.DefaultGameServer(framework.Namespace) + gs.Spec.Template.Spec.Subdomain = "default" + v.setup(gs) + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + require.NoError(t, err) + pod, err := pods.Get(context.Background(), readyGs.ObjectMeta.Name, metav1.GetOptions{}) + require.NoError(t, err) + v.test(readyGs, pod) + }) + } +} + +// nolint:dupl +func TestSDKSetLabel(t *testing.T) { + t.Parallel() + ctx := context.Background() + gs := framework.DefaultGameServer(framework.Namespace) + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + if err != nil { + t.Fatalf("Could not get a GameServer ready: %v", err) + } + + assert.Equal(t, readyGs.Status.State, agonesv1.GameServerStateReady) + reply, err := framework.SendGameServerUDP(t, readyGs, "LABEL") + if err != nil { + t.Fatalf("Could ping GameServer: %v", err) + } + + assert.Equal(t, "ACK: LABEL\n", reply) + + // the label is set in a queue, so it may take a moment + err = wait.PollUntilContextTimeout(ctx, time.Second, 10*time.Second, true, func(ctx context.Context) (bool, error) { + gs, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) + if err != nil { + return true, err + } + return gs.ObjectMeta.Labels != nil, nil + }) + + if assert.NoError(t, err) { + assert.NotEmpty(t, gs.ObjectMeta.Labels["agones.dev/sdk-timestamp"]) + } +} + +func TestHealthCheckDisable(t *testing.T) { + t.Parallel() + ctx := context.Background() + gs := framework.DefaultGameServer(framework.Namespace) + gs.Spec.Health = agonesv1.Health{ + Disabled: true, + FailureThreshold: 1, + InitialDelaySeconds: 1, + PeriodSeconds: 1, + } + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + if err != nil { + t.Fatalf("Could not get a GameServer ready: %v", err) + } + defer framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck + + _, err = framework.SendGameServerUDP(t, readyGs, "UNHEALTHY") + if err != nil { + t.Fatalf("Could not ping GameServer: %v", err) + } + + time.Sleep(10 * time.Second) + + gs, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) + if err != nil { + assert.FailNow(t, "gameserver get failed", err.Error()) + } + + assert.Equal(t, agonesv1.GameServerStateReady, gs.Status.State) +} + +// nolint:dupl +func TestSDKSetAnnotation(t *testing.T) { + t.Parallel() + ctx := context.Background() + gs := framework.DefaultGameServer(framework.Namespace) + annotation := "agones.dev/sdk-timestamp" + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + if err != nil { + t.Fatalf("Could not get a GameServer ready: %v", err) + } + defer framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck + + assert.Equal(t, readyGs.Status.State, agonesv1.GameServerStateReady) + reply, err := framework.SendGameServerUDP(t, readyGs, "ANNOTATION") + if err != nil { + t.Fatalf("Could ping GameServer: %v", err) + } + + assert.Equal(t, "ACK: ANNOTATION\n", reply) + + // the label is set in a queue, so it may take a moment + err = wait.PollUntilContextTimeout(context.Background(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { + gs, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) + if err != nil { + return true, err + } + + _, ok := gs.ObjectMeta.Annotations[annotation] + return ok, nil + }) + + logrus.WithField("annotations", gs.ObjectMeta.Annotations).Info("annotation information") + + if !assert.Nil(t, err) { + assert.FailNow(t, "error waiting on annotation to be set") + } + assert.NotEmpty(t, gs.ObjectMeta.Annotations[annotation]) + assert.NotEmpty(t, gs.ObjectMeta.Annotations[agonesv1.VersionAnnotation]) +} + +func TestUnhealthyGameServerAfterHealthCheckFail(t *testing.T) { + t.Parallel() + gs := framework.DefaultGameServer(framework.Namespace) + gs.Spec.Health.FailureThreshold = 1 + + gs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + if err != nil { + assert.FailNow(t, "Failed to create a gameserver", err.Error()) + } + + reply, err := framework.SendGameServerUDP(t, gs, "UNHEALTHY") + if err != nil { + assert.FailNow(t, "Failed to send a message to a gameserver", err.Error()) + } + assert.Equal(t, "ACK: UNHEALTHY\n", reply) + + _, err = framework.WaitForGameServerState(t, gs, agonesv1.GameServerStateUnhealthy, time.Minute) + assert.NoError(t, err) +} + +func TestUnhealthyGameServersWithoutFreePorts(t *testing.T) { + framework.SkipOnCloudProduct(t, "gke-autopilot", "does not support Static PortPolicy") + if runtime.FeatureEnabled(runtime.FeatureSidecarContainers) { + t.SkipNow() + } + + t.Parallel() + ctx := context.Background() + nodes, err := framework.KubeClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + assert.FailNow(t, "Failed to list nodes", err.Error()) + } + assert.True(t, len(nodes.Items) > 0) + + template := framework.DefaultGameServer(framework.Namespace) + // choose port out of the minport/maxport range + // also rand it, just in case there are still previous static GameServers floating around. + template.Spec.Ports[0].HostPort = int32(rand.IntnRange(4000, 5000)) + template.Spec.Ports[0].PortPolicy = agonesv1.Static + + gameServers := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace) + // one successful static port GameServer + gs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, template.DeepCopy()) + require.NoError(t, err) + + // now let's create the same one, but this time, require it be on the same node. + newGs := template.DeepCopy() + + if newGs.Spec.Template.Spec.NodeSelector == nil { + newGs.Spec.Template.Spec.NodeSelector = map[string]string{} + } + newGs.Spec.Template.Spec.NodeSelector["kubernetes.io/hostname"] = gs.Status.NodeName + newGs, err = gameServers.Create(ctx, newGs, metav1.CreateOptions{}) + require.NoError(t, err) + + _, err = framework.WaitForGameServerState(t, newGs, agonesv1.GameServerStateUnhealthy, 5*time.Minute) + assert.NoError(t, err) +} + +func TestGameServerUnhealthyAfterDeletingPod(t *testing.T) { + t.Parallel() + ctx := context.Background() + gs := framework.DefaultGameServer(framework.Namespace) + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + if err != nil { + t.Fatalf("Could not get a GameServer ready: %v", err) + } + + logrus.WithField("gsKey", readyGs.ObjectMeta.Name).Info("GameServer Ready") + + gsClient := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace) + podClient := framework.KubeClient.CoreV1().Pods(framework.Namespace) + + defer gsClient.Delete(ctx, readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck + + pod, err := podClient.Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) + require.NoError(t, err) + require.True(t, metav1.IsControlledBy(pod, readyGs)) + + err = podClient.Delete(ctx, pod.ObjectMeta.Name, metav1.DeleteOptions{}) + require.NoError(t, err) + + _, err = framework.WaitForGameServerState(t, readyGs, agonesv1.GameServerStateUnhealthy, 3*time.Minute) + require.NoError(t, err) +} + +func TestGameServerRestartBeforeReadyCrash(t *testing.T) { + if runtime.FeatureEnabled(runtime.FeatureSidecarContainers) { + t.SkipNow() + } + + t.Parallel() + ctx := context.Background() + logger := e2eframework.TestLogger(t) + + gs := framework.DefaultGameServer(framework.Namespace) + // give some buffer with gameservers crashing and coming back + gs.Spec.Health.PeriodSeconds = 60 * 60 + gs.Spec.Template.Spec.Containers[0].Env = append(gs.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "READY", Value: "FALSE"}) + gsClient := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace) + newGs, err := gsClient.Create(ctx, gs, metav1.CreateOptions{}) + if err != nil { + assert.Fail(t, "could not create the gameserver", err.Error()) + } + defer gsClient.Delete(ctx, newGs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck + + logger.Info("Waiting for us to have an address to send things to") + newGs, err = framework.WaitForGameServerState(t, newGs, agonesv1.GameServerStateScheduled, framework.WaitForState) + if err != nil { + assert.FailNow(t, "Failed schedule a pod", err.Error()) + } + + logger.WithField("gs", newGs.ObjectMeta.Name).Info("GameServer created") + + address := fmt.Sprintf("%s:%d", newGs.Status.Address, newGs.Status.Ports[0].Port) + logger.WithField("address", address).Info("Dialing UDP message to address") + + messageAndWait := func(gs *agonesv1.GameServer, msg string, check func(gs *agonesv1.GameServer, pod *corev1.Pod) bool) error { + return wait.PollUntilContextTimeout(context.Background(), 200*time.Millisecond, 3*time.Minute, true, func(ctx context.Context) (bool, error) { + gs, err := gsClient.Get(ctx, gs.ObjectMeta.Name, metav1.GetOptions{}) + if err != nil { + logger.WithError(err).Warn("could not get gameserver") + return true, err + } + pod, err := framework.KubeClient.CoreV1().Pods(framework.Namespace).Get(ctx, newGs.ObjectMeta.Name, metav1.GetOptions{}) + if err != nil { + logger.WithError(err).Warn("could not get pod for gameserver") + return true, err + } + + if check(gs, pod) { + return true, nil + } + + // create a connection each time, as weird stuff happens if the receiver isn't up and running. + conn, err := net.Dial("udp", address) + if err != nil { + logger.WithError(err).Warn("could not create connection") + return true, err + } + defer conn.Close() // nolint: errcheck + // doing this last, so that there is a short delay between the msg being sent, and the check. + logger.WithField("gs", gs.ObjectMeta.Name).WithField("msg", msg). + WithField("state", gs.Status.State).Info("sending message") + if _, err = conn.Write([]byte(msg)); err != nil { + logger.WithError(err).WithField("gs", gs.ObjectMeta.Name). + WithField("state", gs.Status.State).Info("error sending packet") + } + return false, nil + }) + } + + logger.Info("crashing, and waiting to see restart") + err = messageAndWait(newGs, "CRASH", func(_ *agonesv1.GameServer, pod *corev1.Pod) bool { + for _, c := range pod.Status.ContainerStatuses { + if c.Name == newGs.Spec.Container && c.RestartCount > 0 { + logger.Info("successfully crashed. Moving on!") + return true + } + } + return false + }) + assert.NoError(t, err) + + // check that the GameServer is not in an unhealthy state. If it does happen, it should happen pretty quick. + // We wait an extra 5s to close the kubelet race in #2445. + newGs, err = framework.WaitForGameServerState(t, newGs, agonesv1.GameServerStateUnhealthy, 10*time.Second) + // should be an error, as the state should not occur + if !assert.Error(t, err) { + assert.FailNow(t, "GameServer should not be Unhealthy") + } + assert.Contains(t, err.Error(), "waiting for GameServer") + + // ping READY until it doesn't fail anymore - since it may take a while + // for this to come back up -- or we could get a delayed CRASH, so we have to + // wait for the process to restart again to fire the SDK.Ready() + logger.Info("marking GameServer as ready") + err = messageAndWait(newGs, "READY", func(gs *agonesv1.GameServer, _ *corev1.Pod) bool { + if gs.Status.State == agonesv1.GameServerStateReady { + logger.Info("ready! Moving On!") + return true + } + return false + }) + assert.NoError(t, err) + + // now crash, should be unhealthy, since it's after being Ready + logger.Info("crashing again, should be unhealthy") + // retry on crash, as with the restarts, sometimes Go takes a moment to send this through. + err = messageAndWait(newGs, "CRASH", func(gs *agonesv1.GameServer, _ *corev1.Pod) bool { + logger.WithField("gs", gs.ObjectMeta.Name).WithField("state", gs.Status.State). + Info("checking final crash state") + if gs.Status.State == agonesv1.GameServerStateUnhealthy { + logger.Info("Unhealthy! We are done!") + return true + } + return false + }) + assert.NoError(t, err) +} + +func TestGameServerUnhealthyAfterReadyCrash(t *testing.T) { + t.Parallel() + ctx := context.Background() + log := e2eframework.TestLogger(t) + + gs := framework.DefaultGameServer(framework.Namespace) + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + if err != nil { + t.Fatalf("Could not get a GameServer ready: %v", err) + } + + log.WithField("gs", readyGs.ObjectMeta.Name).Info("GameServer created") + + gsClient := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace) + defer gsClient.Delete(ctx, readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck + + address := fmt.Sprintf("%s:%d", readyGs.Status.Address, readyGs.Status.Ports[0].Port) + + // keep crashing, until we move to Unhealthy. Solves potential issues with controller Informer cache + // race conditions in which it has yet to see a GameServer is Ready before the crash. + var stop int32 + defer func() { + atomic.StoreInt32(&stop, 1) + }() + go func() { + for { + if atomic.LoadInt32(&stop) > 0 { + log.Info("UDP Crash stop signal received. Stopping.") + return + } + var writeErr error + func() { + conn, err := net.Dial("udp", address) + assert.NoError(t, err) + defer conn.Close() // nolint: errcheck + _, writeErr = conn.Write([]byte("CRASH")) + }() + if writeErr != nil { + log.WithError(err).Warn("error sending udp packet. Stopping.") + return + } + log.WithField("address", address).Info("sent UDP packet") + time.Sleep(5 * time.Second) + } + }() + _, err = framework.WaitForGameServerState(t, readyGs, agonesv1.GameServerStateUnhealthy, 3*time.Minute) + assert.NoError(t, err) +} + +func TestGameServerPodCompletedAfterCleanExit(t *testing.T) { + if !runtime.FeatureEnabled(runtime.FeatureSidecarContainers) { + t.SkipNow() + } + + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + log := e2eframework.TestLogger(t) + + gs := framework.DefaultGameServer(framework.Namespace) + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + if err != nil { + t.Fatalf("Could not get a GameServer ready: %v", err) + } + + address := fmt.Sprintf("%s:%d", readyGs.Status.Address, readyGs.Status.Ports[0].Port) + conn, err := net.Dial("udp", address) + mtx := &sync.Mutex{} + require.NoError(t, err) + defer func() { + mtx.Lock() + defer mtx.Unlock() + if conn != nil { + err = conn.Close() + if err != nil { + log.WithError(err).Warn("error closing udp connection") + } + } + }() + + go func() { + for { + select { + case <-ctx.Done(): + log.Info("Stopping udp CRASH sender") + return + default: + mtx.Lock() + log.Info("Sending CRASH 0") + _, err := conn.Write([]byte("CRASH 0")) + mtx.Unlock() + if err != nil { + log.WithError(err).Warn("error sending udp packet.") + } + } + time.Sleep(5 * time.Second) + } + }() + + result := assert.EventuallyWithT(t, func(c *assert.CollectT) { + gs, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) + log.WithField("gs", readyGs.ObjectMeta.Name).WithField("deletionTimestamp", gs.ObjectMeta.DeletionTimestamp).WithField("state", gs.Status.State).WithError(err).Info("checking if GameServer exists") + assert.True(c, k8serrors.IsNotFound(err)) + }, 5*time.Minute, 3*time.Second) + // debug a lot if it doesn't work, because why! + if !result { + framework.LogEvents(t, log, readyGs.ObjectMeta.Namespace, readyGs) + pod, err := framework.KubeClient.CoreV1().Pods(readyGs.ObjectMeta.Namespace).Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) + if err != nil { + log.WithError(err).Warn("error getting pod for GameServer, skipping debug output") + return + } + log.WithField("metadata", pod.ObjectMeta).WithField("status", pod.Status) + framework.LogEvents(t, log, readyGs.ObjectMeta.Namespace, pod) + framework.LogPodContainers(t, pod) + } +} + +func TestDevelopmentGameServerLifecycle(t *testing.T) { + t.Parallel() + ctx := context.Background() + + labels := map[string]string{"development": "true"} + gs := &agonesv1.GameServer{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "udp-server", + Namespace: framework.Namespace, + Annotations: map[string]string{agonesv1.DevAddressAnnotation: fakeIPAddress}, + Labels: labels, + }, + Spec: agonesv1.GameServerSpec{ + Container: "udp-server", + Ports: []agonesv1.GameServerPort{{ + ContainerPort: 7654, + HostPort: 7654, + Name: "gameport", + PortPolicy: agonesv1.Static, + Protocol: corev1.ProtocolUDP, + }}, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "placebo", + Image: "this is ignored", + }}, + }, + }, + }, + } + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs.DeepCopy()) + if err != nil { + t.Fatalf("Could not get a GameServer ready: %v", err) + } + require.Equal(t, agonesv1.GameServerStateReady, readyGs.Status.State) + + // Set dev GS into RequestReady and confirm it goes back to Ready. + gsCopy := readyGs.DeepCopy() + gsCopy.Status.State = agonesv1.GameServerStateRequestReady + reqReadyGs, err := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Update(ctx, gsCopy, metav1.UpdateOptions{}) + require.NoError(t, err) + require.Equal(t, agonesv1.GameServerStateRequestReady, reqReadyGs.Status.State) + + readyGs, err = framework.WaitForGameServerState(t, reqReadyGs, agonesv1.GameServerStateReady, framework.WaitForState) + if err != nil { + t.Fatalf("Could not get a GameServer ready from request ready: %v", err) + } + require.Equal(t, agonesv1.GameServerStateReady, readyGs.Status.State) + + // confirm delete works, because if the finalisers don't get removed, this won't work. + err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) + require.NoError(t, err) + + err = wait.PollUntilContextTimeout(context.Background(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { + _, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + return true, nil + } + + return false, err + }) + require.NoError(t, err) + + // let's make sure we can allocate a dev gameserver + readyGs, err = framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs.DeepCopy()) + require.NoError(t, err) + + gsa := &allocationv1.GameServerAllocation{ + Spec: allocationv1.GameServerAllocationSpec{ + Selectors: []allocationv1.GameServerSelector{{ + LabelSelector: metav1.LabelSelector{MatchLabels: labels}, + }}, + }, + } + gsa, err = framework.AgonesClient.AllocationV1().GameServerAllocations(framework.Namespace).Create(ctx, gsa, metav1.CreateOptions{}) + require.NoError(t, err) + + require.Equal(t, readyGs.ObjectMeta.Name, gsa.Status.GameServerName) + + _, err = framework.WaitForGameServerState(t, readyGs, agonesv1.GameServerStateAllocated, time.Minute) + require.NoError(t, err) + + // Also confirm that delete works for Allocated state, it should be fine but let's be sure. + err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) + require.NoError(t, err) + + err = wait.PollUntilContextTimeout(context.Background(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { + _, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + return true, nil + } + + return false, err + }) + require.NoError(t, err) +} + +func TestGameServerSelfAllocate(t *testing.T) { + t.Parallel() + ctx := context.Background() + gs := framework.DefaultGameServer(framework.Namespace) + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + if err != nil { + t.Fatalf("Could not get a GameServer ready: %v", err) + } + defer framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck + + assert.Equal(t, readyGs.Status.State, agonesv1.GameServerStateReady) + reply, err := framework.SendGameServerUDP(t, readyGs, "ALLOCATE") + if err != nil { + t.Fatalf("Could not message GameServer: %v", err) + } + + assert.Equal(t, "ACK: ALLOCATE\n", reply) + gs, err = framework.WaitForGameServerState(t, readyGs, agonesv1.GameServerStateAllocated, time.Minute) + if assert.NoError(t, err) { + assert.Equal(t, agonesv1.GameServerStateAllocated, gs.Status.State) + } +} + +func TestGameServerReadyAllocateReady(t *testing.T) { + t.Parallel() + ctx := context.Background() + logger := e2eframework.TestLogger(t) + + gs := framework.DefaultGameServer(framework.Namespace) + + logger.Info("Moving to Ready") + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + require.NoError(t, err, "Could not get a GameServer ready") + logger = logger.WithField("gs", readyGs.ObjectMeta.Name) + + defer framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck + + require.Equal(t, readyGs.Status.State, agonesv1.GameServerStateReady) + + logger.Info("Moving to Allocated") + reply, err := framework.SendGameServerUDP(t, readyGs, "ALLOCATE") + require.NoError(t, err, "Could not message GameServer") + + require.Equal(t, "ACK: ALLOCATE\n", reply) + gs, err = framework.WaitForGameServerState(t, readyGs, agonesv1.GameServerStateAllocated, time.Minute) + require.NoError(t, err) + require.Equal(t, agonesv1.GameServerStateAllocated, gs.Status.State) + + logger.Info("Moving to Ready again") + reply, err = framework.SendGameServerUDP(t, readyGs, "READY") + require.NoError(t, err, "Could not message GameServer") + require.Equal(t, "ACK: READY\n", reply) + gs, err = framework.WaitForGameServerState(t, gs, agonesv1.GameServerStateReady, time.Minute) + require.NoError(t, err) + require.Equal(t, agonesv1.GameServerStateReady, gs.Status.State) +} + +func TestGameServerWithPortsMappedToMultipleContainers(t *testing.T) { + t.Parallel() + ctx := context.Background() + + firstContainerName := "udp-server" + secondContainerName := "second-udp-server" + firstPort := "gameport" + secondPort := "second-gameport" + gs := &agonesv1.GameServer{ + ObjectMeta: metav1.ObjectMeta{GenerateName: "udp-server", Namespace: framework.Namespace}, + Spec: agonesv1.GameServerSpec{ + Container: firstContainerName, + Ports: []agonesv1.GameServerPort{{ + ContainerPort: 7654, + Name: firstPort, + PortPolicy: agonesv1.Dynamic, + Protocol: corev1.ProtocolUDP, + }, { + ContainerPort: 5000, + Name: secondPort, + PortPolicy: agonesv1.Dynamic, + Protocol: corev1.ProtocolUDP, + Container: &secondContainerName, + }}, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: firstContainerName, + Image: framework.GameServerImage, + ImagePullPolicy: corev1.PullIfNotPresent, + }, + { + Name: secondContainerName, + Image: framework.GameServerImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{"-port", "5000"}, + }, + }, + }, + }, + }, + } + + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + if err != nil { + t.Fatalf("Could not get a GameServer ready: %v", err) + } + defer framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck + assert.Equal(t, readyGs.Status.State, agonesv1.GameServerStateReady) + + interval := 2 * time.Second + timeOut := 60 * time.Second + + expectedMsg1 := "Ping 1" + errPoll := wait.PollUntilContextTimeout(context.Background(), interval, timeOut, true, func(_ context.Context) (done bool, err error) { + res, err := framework.SendGameServerUDPToPort(t, readyGs, firstPort, expectedMsg1) + if err != nil { + t.Logf("Could not message GameServer on %s: %v. Will try again...", firstPort, err) + } + return err == nil && strings.Contains(res, expectedMsg1), nil + }) + if errPoll != nil { + assert.FailNow(t, errPoll.Error(), "expected no errors after polling a port: %s", firstPort) + } + + expectedMsg2 := "Ping 2" + errPoll = wait.PollUntilContextTimeout(context.Background(), interval, timeOut, true, func(_ context.Context) (done bool, err error) { + res, err := framework.SendGameServerUDPToPort(t, readyGs, secondPort, expectedMsg2) + if err != nil { + t.Logf("Could not message GameServer on %s: %v. Will try again...", secondPort, err) + } + return err == nil && strings.Contains(res, expectedMsg2), nil + }) + + assert.NoError(t, errPoll, "expected no errors after polling a port: %s", secondPort) +} + +func TestGameServerWithPortsMappedToInitSidecarContainers(t *testing.T) { + t.Skip("Skip test. Only works on some CNIs. The current e2e platform does not support ports on init containers.") + + t.Parallel() + ctx := context.Background() + + containerName := "udp-server" + sidecarContainerName := "sidecar-server" + firstPort := "gameport" + secondPort := "second-gameport" + gs := &agonesv1.GameServer{ + ObjectMeta: metav1.ObjectMeta{GenerateName: "udp-server", Namespace: framework.Namespace}, + Spec: agonesv1.GameServerSpec{ + Container: containerName, + Ports: []agonesv1.GameServerPort{{ + ContainerPort: 7654, + Name: firstPort, + PortPolicy: agonesv1.Dynamic, + Protocol: corev1.ProtocolUDP, + }, { + ContainerPort: 5000, + Name: secondPort, + PortPolicy: agonesv1.Dynamic, + Protocol: corev1.ProtocolUDP, + Container: &sidecarContainerName, + }}, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: containerName, + Image: framework.GameServerImage, + ImagePullPolicy: corev1.PullIfNotPresent, + }, + }, + InitContainers: []corev1.Container{ + { + Name: sidecarContainerName, + Image: framework.GameServerImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{"-port", "5000"}, + RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways), + }, + }, + }, + }, + }, + } + + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + if err != nil { + t.Fatalf("Could not get a GameServer ready: %v", err) + } + defer framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck + assert.Equal(t, readyGs.Status.State, agonesv1.GameServerStateReady) + + interval := 2 * time.Second + timeOut := 60 * time.Second + + expectedMsg1 := "Ping 1" + assert.EventuallyWithT(t, func(c *assert.CollectT) { + res, err := framework.SendGameServerUDPToPort(t, readyGs, firstPort, expectedMsg1) + require.NoError(c, err, "expected no errors after messaging GameServer a port: %s", firstPort) + assert.Contains(c, res, expectedMsg1) + }, timeOut, interval) + + expectedMsg2 := "Ping 2" + assert.EventuallyWithT(t, func(c *assert.CollectT) { + res, err := framework.SendGameServerUDPToPort(t, readyGs, secondPort, expectedMsg2) + require.NoError(c, err, "expected no errors after messaging GameServer a port: %s", secondPort) + assert.Contains(c, res, expectedMsg2) + }, timeOut, interval) +} + +func TestGameServerReserve(t *testing.T) { + t.Parallel() + ctx := context.Background() + + // We are deliberately not trying to test the transition between Reserved -> Ready. + // + // We have found that trying to catch the GameServer in the Reserved state can be flaky, + // as we can't control the speed in which the Kubernetes API is going to reply to request, + // and we could sometimes miss when the GameServer is in the Reserved State before it goes to Ready. + // + // Therefore we are going to test for concrete states that we don't need to catch while + // in a transitive state. + + gs := framework.DefaultGameServer(framework.Namespace) + gs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + if err != nil { + assert.FailNow(t, "Could not get a GameServer ready", err.Error()) + } + defer framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, gs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck + assert.Equal(t, gs.Status.State, agonesv1.GameServerStateReady) + + reply, err := framework.SendGameServerUDP(t, gs, "RESERVE 0") + if err != nil { + assert.FailNow(t, "Could not message GameServer", err.Error()) + } + assert.Equal(t, "ACK: RESERVE 0\n", reply) + + gs, err = framework.WaitForGameServerState(t, gs, agonesv1.GameServerStateReserved, 3*time.Minute) + if err != nil { + assert.FailNow(t, "Time out on waiting for gs in a Reserved state", err.Error()) + } + + reply, err = framework.SendGameServerUDP(t, gs, "ALLOCATE") + if err != nil { + assert.FailNow(t, "Could not message GameServer", err.Error()) + } + assert.Equal(t, "ACK: ALLOCATE\n", reply) + + // put it in a totally different state, just to reset things. + gs, err = framework.WaitForGameServerState(t, gs, agonesv1.GameServerStateAllocated, 3*time.Minute) + if err != nil { + assert.FailNow(t, "Time out on waiting for gs in an Allocated state", err.Error()) + } + + reply, err = framework.SendGameServerUDP(t, gs, "RESERVE 5s") + if err != nil { + assert.FailNow(t, "Could not message GameServer", err.Error()) + } + assert.Equal(t, "ACK: RESERVE 5s\n", reply) + + // sleep, since we're going to wait for the Ready response. + time.Sleep(5 * time.Second) + _, err = framework.WaitForGameServerState(t, gs, agonesv1.GameServerStateReady, 3*time.Minute) + assert.NoError(t, err) +} + +func TestGameServerShutdown(t *testing.T) { + t.Parallel() + ctx := context.Background() + gs := framework.DefaultGameServer(framework.Namespace) + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + if err != nil { + t.Fatalf("Could not get a GameServer ready: %v", err) + } + assert.Equal(t, readyGs.Status.State, agonesv1.GameServerStateReady) + + reply, err := framework.SendGameServerUDP(t, readyGs, "EXIT") + if err != nil { + t.Fatalf("Could not message GameServer: %v", err) + } + + assert.Equal(t, "ACK: EXIT\n", reply) + + err = wait.PollUntilContextTimeout(ctx, time.Second, 3*time.Minute, true, func(ctx context.Context) (bool, error) { + gs, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) + + if k8serrors.IsNotFound(err) { + return true, nil + } + + return false, err + }) + + assert.NoError(t, err) +} + +// TestGameServerEvicted test that if Gameserver would be evicted than it becomes Unhealthy +// Ephemeral Storage limit set to 0Mi +func TestGameServerEvicted(t *testing.T) { + t.Parallel() + log := e2eframework.TestLogger(t) + + ctx := context.Background() + gs := framework.DefaultGameServer(framework.Namespace) + newGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + require.NoError(t, err) + log.WithField("name", newGs.ObjectMeta.Name).Info("GameServer created, waiting for being Evicted and Unhealthy") + defer framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, newGs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck + + pods := framework.KubeClient.CoreV1().Pods(framework.Namespace) + pod, err := pods.Get(ctx, newGs.ObjectMeta.Name, metav1.GetOptions{}) + require.NoError(t, err) + + eviction := &policyv1.Eviction{ + ObjectMeta: metav1.ObjectMeta{ + Name: pod.Name, + Namespace: pod.Namespace, + }, + } + go func() { + time.Sleep(3 * time.Second) // just make sure it comes in later + log.WithField("name", eviction.ObjectMeta.Name).Info("Evicting pod!") + err := pods.EvictV1(context.Background(), eviction) + require.NoError(t, err) + }() + + _, err = framework.WaitForGameServerState(t, newGs, agonesv1.GameServerStateUnhealthy, 10*time.Minute) + require.NoError(t, err, fmt.Sprintf("waiting for [%v] GameServer Unhealthy state timed out (%v)", gs.Status.State, gs.Name)) +} + +func TestGameServerPassthroughPort(t *testing.T) { + framework.SkipOnCloudProduct(t, "gke-autopilot", "does not support Passthrough PortPolicy") + t.Parallel() + gs := framework.DefaultGameServer(framework.Namespace) + gs.Spec.Ports[0] = agonesv1.GameServerPort{PortPolicy: agonesv1.Passthrough} + gs.Spec.Template.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "PASSTHROUGH", Value: "TRUE"}} + // gate + errs := gs.Validate(agtesting.FakeAPIHooks{}) + assert.Len(t, errs, 0) + + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + if err != nil { + assert.FailNow(t, "Could not get a GameServer ready", err.Error()) + } + + port := readyGs.Spec.Ports[0] + assert.Equal(t, agonesv1.Passthrough, port.PortPolicy) + assert.NotEmpty(t, port.HostPort) + assert.Equal(t, port.HostPort, port.ContainerPort) + + reply, err := framework.SendGameServerUDP(t, readyGs, "Hello World !") + if err != nil { + t.Fatalf("Could ping GameServer: %v", err) + } + + assert.Equal(t, "ACK: Hello World !\n", reply) +} + +func TestGameServerPortPolicyNone(t *testing.T) { + + t.Parallel() + + gs := framework.DefaultGameServer(framework.Namespace) + gs.Spec.Ports[0] = agonesv1.GameServerPort{PortPolicy: agonesv1.None, ContainerPort: 7777} + // gate + errs := gs.Validate(agtesting.FakeAPIHooks{}) + assert.Len(t, errs, 0) + + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + if err != nil { + assert.FailNow(t, "Could not get a GameServer ready", err.Error()) + } + + // To test sending UDP traffic directly to a pod when no hostPort is set, a product like Calico which uses BGP is needed + // so this only tests that the port is set correctly. + port := readyGs.Spec.Ports[0] + assert.Equal(t, agonesv1.None, port.PortPolicy) + assert.Empty(t, port.HostPort) + assert.EqualValues(t, 7777, port.ContainerPort) +} + +func TestGameServerTcpProtocol(t *testing.T) { + t.Parallel() + log := e2eframework.TestLogger(t) + ctx := context.Background() + gs := framework.DefaultGameServer(framework.Namespace) + + gs.Spec.Ports[0].Protocol = corev1.ProtocolTCP + gs.Spec.Template.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "TCP", Value: "TRUE"}} + + errs := gs.Validate(agtesting.FakeAPIHooks{}) + require.Len(t, errs, 0) + + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + require.NoError(t, err) + + replyTCP, err := e2eframework.SendGameServerTCP(readyGs, "Hello World !") + if err != nil { + framework.LogEvents(t, log, readyGs.ObjectMeta.Namespace, readyGs) + pod, err := framework.KubeClient.CoreV1().Pods(readyGs.ObjectMeta.Namespace).Get(ctx, readyGs.Name, metav1.GetOptions{}) + if err != nil { + log.WithError(err).Info("Could not retrieve pod for GameServer") + } else { + framework.LogEvents(t, log, readyGs.ObjectMeta.Namespace, pod) + framework.LogPodContainers(t, pod) + } + } + require.NoError(t, err) + assert.Equal(t, "ACK TCP: Hello World !\n", replyTCP) +} + +func TestGameServerTcpUdpProtocol(t *testing.T) { + t.Parallel() + gs := framework.DefaultGameServer(framework.Namespace) + + gs.Spec.Ports[0].Protocol = agonesv1.ProtocolTCPUDP + gs.Spec.Ports[0].Name = "gameserver" + gs.Spec.Template.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "TCP", Value: "TRUE"}} + + errs := gs.Validate(agtesting.FakeAPIHooks{}) + require.Len(t, errs, 0) + + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + if err != nil { + assert.FailNow(t, "Could not get a GameServer ready", err.Error()) + } + + tcpPort := readyGs.Spec.Ports[0] + assert.Equal(t, corev1.ProtocolTCP, tcpPort.Protocol) + assert.NotEmpty(t, tcpPort.HostPort) + assert.Equal(t, "gameserver-tcp", tcpPort.Name) + + udpPort := readyGs.Spec.Ports[1] + assert.Equal(t, corev1.ProtocolUDP, udpPort.Protocol) + assert.NotEmpty(t, udpPort.HostPort) + assert.Equal(t, "gameserver-udp", udpPort.Name) + + assert.Equal(t, tcpPort.HostPort, udpPort.HostPort) + + logrus.WithField("name", readyGs.ObjectMeta.Name).Info("GameServer created, sending UDP ping") + + replyUDP, err := framework.SendGameServerUDPToPort(t, readyGs, udpPort.Name, "Hello World !") + require.NoError(t, err) + if err != nil { + t.Fatalf("Could not ping UDP GameServer: %v", err) + } + + assert.Equal(t, "ACK: Hello World !\n", replyUDP) + + logrus.WithField("name", readyGs.ObjectMeta.Name).Info("UDP ping passed, sending TCP ping") + + replyTCP, err := e2eframework.SendGameServerTCPToPort(readyGs, tcpPort.Name, "Hello World !") + if err != nil { + t.Fatalf("Could not ping TCP GameServer: %v", err) + } + + assert.Equal(t, "ACK TCP: Hello World !\n", replyTCP) +} + +// TestGameServerStaticTcpUdpProtocol checks UDP and TCP connection for TCPUDP Protocol of Static Portpolicy +func TestGameServerStaticTcpUdpProtocol(t *testing.T) { + framework.SkipOnCloudProduct(t, "gke-autopilot", "does not support Static PortPolicy") + t.Parallel() + gs := framework.DefaultGameServer(framework.Namespace) + gs.Spec.Ports[0].PortPolicy = agonesv1.Static + gs.Spec.Ports[0].Protocol = agonesv1.ProtocolTCPUDP + gs.Spec.Ports[0].HostPort = 7000 + gs.Spec.Ports[0].Name = "gameserver" + gs.Spec.Template.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "TCP", Value: "TRUE"}} + + errs := gs.Validate(agtesting.FakeAPIHooks{}) + require.Len(t, errs, 0) + + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + require.NoError(t, err) + + tcpPort := readyGs.Spec.Ports[0] + assert.Equal(t, corev1.ProtocolTCP, tcpPort.Protocol) + assert.NotEmpty(t, tcpPort.HostPort) + assert.Equal(t, "gameserver-tcp", tcpPort.Name) + + udpPort := readyGs.Spec.Ports[1] + assert.Equal(t, corev1.ProtocolUDP, udpPort.Protocol) + assert.NotEmpty(t, udpPort.HostPort) + assert.Equal(t, "gameserver-udp", udpPort.Name) + + assert.Equal(t, tcpPort.HostPort, udpPort.HostPort) + + logrus.WithField("name", readyGs.ObjectMeta.Name).Info("GameServer created, sending UDP ping") + + replyUDP, err := framework.SendGameServerUDPToPort(t, readyGs, udpPort.Name, "Hello World !") + require.NoError(t, err) + if err != nil { + t.Fatalf("Could not ping UDP GameServer: %v", err) + } + + assert.Equal(t, "ACK: Hello World !\n", replyUDP) + + logrus.WithField("name", readyGs.ObjectMeta.Name).Info("UDP ping passed, sending TCP ping") + + replyTCP, err := e2eframework.SendGameServerTCPToPort(readyGs, tcpPort.Name, "Hello World !") + if err != nil { + t.Fatalf("Could not ping TCP GameServer: %v", err) + } + + assert.Equal(t, "ACK TCP: Hello World !\n", replyTCP) +} + +// TestGameServerStaticTcpProtocol checks TCP connection for TCP Protocol of Static Portpolicy +func TestGameServerStaticTcpProtocol(t *testing.T) { + framework.SkipOnCloudProduct(t, "gke-autopilot", "does not support Static PortPolicy") + t.Parallel() + gs := framework.DefaultGameServer(framework.Namespace) + + gs.Spec.Ports[0].PortPolicy = agonesv1.Static + gs.Spec.Ports[0].Protocol = corev1.ProtocolTCP + gs.Spec.Ports[0].HostPort = 7000 + gs.Spec.Template.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "TCP", Value: "TRUE"}} + + errs := gs.Validate(agtesting.FakeAPIHooks{}) + require.Len(t, errs, 0) + + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + require.NoError(t, err) + + logrus.WithField("name", readyGs.ObjectMeta.Name).Info("sending TCP ping") + + replyTCP, err := e2eframework.SendGameServerTCP(readyGs, "Hello World !") + require.NoError(t, err) + assert.Equal(t, "ACK TCP: Hello World !\n", replyTCP) + + logrus.WithField("name", readyGs.ObjectMeta.Name).Info("TCP ping Passed") +} + +// TestGameServerStaticUdpProtocol checks default(UDP) connection of Static Portpolicy +func TestGameServerStaticUdpProtocol(t *testing.T) { + framework.SkipOnCloudProduct(t, "gke-autopilot", "does not support Static PortPolicy") + t.Parallel() + gs := framework.DefaultGameServer(framework.Namespace) + + gs.Spec.Ports[0].PortPolicy = agonesv1.Static + gs.Spec.Ports[0].HostPort = 7000 + + errs := gs.Validate(agtesting.FakeAPIHooks{}) + require.Len(t, errs, 0) + + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + require.NoError(t, err) + + logrus.WithField("name", readyGs.ObjectMeta.Name).Info("sending UDP ping") + + replyTCP, err := framework.SendGameServerUDP(t, readyGs, "Default UDP connection check") + require.NoError(t, err) + assert.Equal(t, "ACK: Default UDP connection check\n", replyTCP) + + logrus.WithField("name", readyGs.ObjectMeta.Name).Info("UDP ping Passed") +} + +func TestGameServerWithoutPort(t *testing.T) { + t.Parallel() + gs := framework.DefaultGameServer(framework.Namespace) + gs.Spec.Ports = nil + + errs := gs.Validate(agtesting.FakeAPIHooks{}) + assert.Len(t, errs, 0) + + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + + require.NoError(t, err, "Could not get a GameServer ready") + assert.Empty(t, readyGs.Spec.Ports) +} + +// TestGameServerResourceValidation - check that we are not able to use +// invalid PodTemplate for GameServer Spec with wrong Resource Requests and Limits +func TestGameServerResourceValidation(t *testing.T) { + t.Parallel() + ctx := context.Background() + gs := framework.DefaultGameServer(framework.Namespace) + mi64 := resource.MustParse("64Mi") + gs.Spec.Template.Spec.Containers[0].Resources.Limits[corev1.ResourceMemory] = mi64 + gs.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceMemory] = resource.MustParse("128Mi") + + errs := gs.Validate(agtesting.FakeAPIHooks{}) + assert.False(t, len(errs) == 0) + + gsClient := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace) + + _, err := gsClient.Create(ctx, gs.DeepCopy(), metav1.CreateOptions{}) + assert.NotNil(t, err) + statusErr, ok := err.(*k8serrors.StatusError) + assert.True(t, ok) + assert.Len(t, statusErr.Status().Details.Causes, 1) + assert.Equal(t, metav1.CauseTypeFieldValueInvalid, statusErr.Status().Details.Causes[0].Type) + assert.Equal(t, "spec.template.spec.containers[0].resources.requests", statusErr.Status().Details.Causes[0].Field) + + gs.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = resource.MustParse("-50m") + _, err = gsClient.Create(ctx, gs.DeepCopy(), metav1.CreateOptions{}) + assert.NotNil(t, err) + statusErr, ok = err.(*k8serrors.StatusError) + assert.True(t, ok) + assert.Len(t, statusErr.Status().Details.Causes, 2) + sort.Slice(statusErr.Status().Details.Causes, func(i, j int) bool { + return statusErr.Status().Details.Causes[i].Field > statusErr.Status().Details.Causes[j].Field + }) + assert.Equal(t, metav1.CauseTypeFieldValueInvalid, statusErr.Status().Details.Causes[0].Type) + assert.Equal(t, "spec.template.spec.containers[0].resources.requests[cpu]", statusErr.Status().Details.Causes[0].Field) + + // test that values are still being set correctly + m50 := resource.MustParse("50m") + gs.Spec.Template.Spec.Containers[0].Resources.Limits[corev1.ResourceMemory] = mi64 + gs.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceMemory] = mi64 + gs.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = m50 + gs.Spec.Template.Spec.Containers[0].Resources.Limits[corev1.ResourceCPU] = m50 + + // confirm we have a valid GameServer before running the test + errs = gs.Validate(agtesting.FakeAPIHooks{}) + require.Len(t, errs, 0) + + gsCopy, err := gsClient.Create(ctx, gs.DeepCopy(), metav1.CreateOptions{}) + require.NoError(t, err) + assert.Equal(t, mi64, gsCopy.Spec.Template.Spec.Containers[0].Resources.Limits[corev1.ResourceMemory]) + assert.Equal(t, mi64, gsCopy.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceMemory]) + assert.Equal(t, m50, gsCopy.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU]) + assert.Equal(t, m50, gsCopy.Spec.Template.Spec.Containers[0].Resources.Limits[corev1.ResourceCPU]) +} + +func TestGameServerPodTemplateSpecFailSchemaValidation(t *testing.T) { + t.Parallel() + + // The Kubernetes dynamic client skips schema validation (for reasons I'm not sure of), so the + // best way I could determine to test schema validation is via calling kubectl directly. + // The schema's from Kubernetes don't include anything like `pattern:` or `enum:` which would + // potentially make this easier to test. + + gsYaml := ` +apiVersion: "agones.dev/v1" +kind: GameServer +metadata: + name: "invalid-gameserver" +spec: + ports: + - name: default + portPolicy: Dynamic + containerPort: 7654 + template: + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: ERROR + containers: + - name: simple-game-server + image: us-docker.pkg.dev/agones-images/examples/simple-game-server:0.43 +` + err := os.WriteFile("/tmp/invalid.yaml", []byte(gsYaml), 0o644) + require.NoError(t, err) + + cmd := exec.Command("kubectl", "apply", "-f", "/tmp/invalid.yaml") + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err = cmd.Run() + logrus.WithField("stdout", stdout.String()).WithField("stderr", stderr.String()).WithError(err).Info("Ran command!") + require.Error(t, err) + assert.Contains(t, stderr.String(), "spec.template.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution") +} + +func TestGameServerSetPlayerCapacity(t *testing.T) { + if !runtime.FeatureEnabled(runtime.FeaturePlayerTracking) { + t.SkipNow() + } + t.Parallel() + ctx := context.Background() + + t.Run("no initial capacity set", func(t *testing.T) { + gs := framework.DefaultGameServer(framework.Namespace) + gs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + if err != nil { + t.Fatalf("Could not get a GameServer ready: %v", err) + } + assert.Equal(t, gs.Status.State, agonesv1.GameServerStateReady) + assert.Equal(t, int64(0), gs.Status.Players.Capacity) + + reply, err := framework.SendGameServerUDP(t, gs, "PLAYER_CAPACITY") + if err != nil { + t.Fatalf("Could not message GameServer: %v", err) + } + assert.Equal(t, "0\n", reply) + + reply, err = framework.SendGameServerUDP(t, gs, "PLAYER_CAPACITY 20") + if err != nil { + t.Fatalf("Could not message GameServer: %v", err) + } + assert.Equal(t, "ACK: PLAYER_CAPACITY 20\n", reply) + + reply, err = framework.SendGameServerUDP(t, gs, "PLAYER_CAPACITY") + if err != nil { + t.Fatalf("Could not message GameServer: %v", err) + } + assert.Equal(t, "20\n", reply) + + err = wait.PollUntilContextTimeout(ctx, time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { + gs, err := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, gs.ObjectMeta.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + return gs.Status.Players.Capacity == 20, nil + }) + assert.NoError(t, err) + }) + + t.Run("initial capacity set", func(t *testing.T) { + gs := framework.DefaultGameServer(framework.Namespace) + gs.Spec.Players = &agonesv1.PlayersSpec{InitialCapacity: 10} + gs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + if err != nil { + t.Fatalf("Could not get a GameServer ready: %v", err) + } + assert.Equal(t, gs.Status.State, agonesv1.GameServerStateReady) + assert.Equal(t, int64(10), gs.Status.Players.Capacity) + + reply, err := framework.SendGameServerUDP(t, gs, "PLAYER_CAPACITY") + if err != nil { + t.Fatalf("Could not message GameServer: %v", err) + } + assert.Equal(t, "10\n", reply) + + reply, err = framework.SendGameServerUDP(t, gs, "PLAYER_CAPACITY 20") + if err != nil { + t.Fatalf("Could not message GameServer: %v", err) + } + assert.Equal(t, "ACK: PLAYER_CAPACITY 20\n", reply) + + reply, err = framework.SendGameServerUDP(t, gs, "PLAYER_CAPACITY") + if err != nil { + t.Fatalf("Could not message GameServer: %v", err) + } + assert.Equal(t, "20\n", reply) + + err = wait.PollUntilContextTimeout(context.Background(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { + gs, err := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, gs.ObjectMeta.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + return gs.Status.Players.Capacity == 20, nil + }) + assert.NoError(t, err) + + time.Sleep(30 * time.Second) + }) +} + +func TestPlayerConnectWithCapacityZero(t *testing.T) { + if !runtime.FeatureEnabled(runtime.FeaturePlayerTracking) { + t.SkipNow() + } + t.Parallel() + + gs := framework.DefaultGameServer(framework.Namespace) + playerCount := int64(0) + gs.Spec.Players = &agonesv1.PlayersSpec{InitialCapacity: playerCount} + gs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + require.NoError(t, err) + assert.Equal(t, gs.Status.State, agonesv1.GameServerStateReady) + assert.Equal(t, playerCount, gs.Status.Players.Capacity) + + // add a player + msg := "PLAYER_CONNECT 1" + logrus.WithField("msg", msg).Info("Sending Player Connect") + _, err = framework.SendGameServerUDP(t, gs, msg) + // expected error from the log.Fatalf("could not connect player: %v", err) + if assert.Error(t, err) { + _, err := framework.WaitForGameServerState(t, gs, agonesv1.GameServerStateUnhealthy, time.Minute) + assert.NoError(t, err) + } +} + +func TestPlayerConnectAndDisconnect(t *testing.T) { + if !runtime.FeatureEnabled(runtime.FeaturePlayerTracking) { + t.SkipNow() + } + t.Parallel() + ctx := context.Background() + + gs := framework.DefaultGameServer(framework.Namespace) + playerCount := int64(3) + gs.Spec.Players = &agonesv1.PlayersSpec{InitialCapacity: playerCount} + gs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + if err != nil { + t.Fatalf("Could not get a GameServer ready: %v", err) + } + assert.Equal(t, gs.Status.State, agonesv1.GameServerStateReady) + assert.Equal(t, playerCount, gs.Status.Players.Capacity) + + // add three players in quick succession + for i := int64(1); i <= playerCount; i++ { + msg := "PLAYER_CONNECT " + fmt.Sprintf("%d", i) + logrus.WithField("msg", msg).Info("Sending Player Connect") + reply, err := framework.SendGameServerUDP(t, gs, msg) + if err != nil { + t.Fatalf("Could not message GameServer: %v", err) + } + assert.Equal(t, fmt.Sprintf("ACK: %s\n", msg), reply) + } + + // deliberately do this before polling, to test the SDK returning the correct + // results before it is committed to the GameServer resource. + reply, err := framework.SendGameServerUDP(t, gs, "PLAYER_CONNECTED 1") + if err != nil { + t.Fatalf("Could not message GameServer: %v", err) + } + assert.Equal(t, "true\n", reply) + + reply, err = framework.SendGameServerUDP(t, gs, "GET_PLAYERS") + if err != nil { + t.Fatalf("Could not message GameServer: %v", err) + } + assert.ElementsMatch(t, []string{"1", "2", "3"}, strings.Split(strings.TrimSpace(reply), ",")) + + reply, err = framework.SendGameServerUDP(t, gs, "PLAYER_COUNT") + if err != nil { + t.Fatalf("Could not message GameServer: %v", err) + } + assert.Equal(t, "3\n", reply) + + err = wait.PollUntilContextTimeout(ctx, time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { + gs, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, gs.ObjectMeta.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + return gs.Status.Players.Count == playerCount, nil + }) + assert.NoError(t, err) + assert.ElementsMatch(t, []string{"1", "2", "3"}, gs.Status.Players.IDs) + + // let's disconnect player 2 + logrus.Info("Disconnect Player 2") + reply, err = framework.SendGameServerUDP(t, gs, "PLAYER_DISCONNECT 2") + if err != nil { + t.Fatalf("Could not message GameServer: %v", err) + } + assert.Equal(t, "ACK: PLAYER_DISCONNECT 2\n", reply) + + reply, err = framework.SendGameServerUDP(t, gs, "PLAYER_CONNECTED 2") + if err != nil { + t.Fatalf("Could not message GameServer: %v", err) + } + assert.Equal(t, "false\n", reply) + + reply, err = framework.SendGameServerUDP(t, gs, "GET_PLAYERS") + if err != nil { + t.Fatalf("Could not message GameServer: %v", err) + } + assert.ElementsMatch(t, []string{"1", "3"}, strings.Split(strings.TrimSpace(reply), ",")) + + reply, err = framework.SendGameServerUDP(t, gs, "PLAYER_COUNT") + if err != nil { + t.Fatalf("Could not message GameServer: %v", err) + } + assert.Equal(t, "2\n", reply) + + err = wait.PollUntilContextTimeout(ctx, time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { + gs, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, gs.ObjectMeta.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + return gs.Status.Players.Count == 2, nil + }) + assert.NoError(t, err) + assert.ElementsMatch(t, []string{"1", "3"}, gs.Status.Players.IDs) +} + +func TestCounters(t *testing.T) { + if !runtime.FeatureEnabled(runtime.FeatureCountsAndLists) { + t.SkipNow() + } + t.Parallel() + ctx := context.Background() + gs := framework.DefaultGameServer(framework.Namespace) + + gs.Spec.Counters = make(map[string]agonesv1.CounterStatus) + gs.Spec.Counters["games"] = agonesv1.CounterStatus{ + Count: 1, + Capacity: 50, + } + gs.Spec.Counters["foo"] = agonesv1.CounterStatus{ + Count: 10, + Capacity: 100, + } + gs.Spec.Counters["bar"] = agonesv1.CounterStatus{ + Count: 10, + Capacity: 10, + } + gs.Spec.Counters["baz"] = agonesv1.CounterStatus{ + Count: 1000, + Capacity: 1000, + } + gs.Spec.Counters["qux"] = agonesv1.CounterStatus{ + Count: 42, + Capacity: 50, + } + + gs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + require.NoError(t, err) + defer framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, gs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck + assert.Equal(t, agonesv1.GameServerStateReady, gs.Status.State) + + testCases := map[string]struct { + msg string + want string + counterName string + wantCount string + wantCapacity string + }{ + "GetCounterCount": { + msg: "GET_COUNTER_COUNT games", + want: "COUNTER: 1\n", + }, + "GetCounterCount Counter Does Not Exist": { + msg: "GET_COUNTER_COUNT fame", + want: "ERROR: -1\n", + }, + "IncrementCounter": { + msg: "INCREMENT_COUNTER foo 10", + want: "SUCCESS\n", + counterName: "foo", + wantCount: "COUNTER: 20\n", + }, + "IncrementCounter Past Capacity": { + msg: "INCREMENT_COUNTER games 50", + want: "ERROR: could not increment Counter games by amount 50: rpc error: code = Unknown desc = out of range. Count must be within range [0,Capacity]. Found Count: 51, Capacity: 50\n", + counterName: "games", + wantCount: "COUNTER: 1\n", + }, + "IncrementCounter Negative": { + msg: "INCREMENT_COUNTER games -1", + want: "ERROR: amount must be a positive int64, found -1\n", + counterName: "games", + wantCount: "COUNTER: 1\n", + }, + "IncrementCounter Counter Does Not Exist": { + msg: "INCREMENT_COUNTER same 1", + want: "ERROR: could not increment Counter same by amount 1: rpc error: code = Unknown desc = counter not found: same\n", + }, + "DecrementCounter": { + msg: "DECREMENT_COUNTER bar 10", + want: "SUCCESS\n", + counterName: "bar", + wantCount: "COUNTER: 0\n", + }, + "DecrementCounter Past Capacity": { + msg: "DECREMENT_COUNTER games 2", + want: "ERROR: could not decrement Counter games by amount 2: rpc error: code = Unknown desc = out of range. Count must be within range [0,Capacity]. Found Count: -1, Capacity: 50\n", + counterName: "games", + wantCount: "COUNTER: 1\n", + }, + "DecrementCounter Negative": { + msg: "DECREMENT_COUNTER games -1", + want: "ERROR: amount must be a positive int64, found -1\n", + counterName: "games", + wantCount: "COUNTER: 1\n", + }, + "DecrementCounter Counter Does Not Exist": { + msg: "DECREMENT_COUNTER lame 1", + want: "ERROR: could not decrement Counter lame by amount 1: rpc error: code = Unknown desc = counter not found: lame\n", + }, + "SetCounterCount": { + msg: "SET_COUNTER_COUNT baz 0", + want: "SUCCESS\n", + counterName: "baz", + wantCount: "COUNTER: 0\n", + }, + "SetCounterCount Past Capacity": { + msg: "SET_COUNTER_COUNT games 51", + want: "ERROR: could not set Counter games count to amount 51: rpc error: code = Unknown desc = out of range. Count must be within range [0,Capacity]. Found Count: 51, Capacity: 50\n", + counterName: "games", + wantCount: "COUNTER: 1\n", + }, + "SetCounterCount Past Zero": { + msg: "SET_COUNTER_COUNT games -1", + want: "ERROR: could not set Counter games count to amount -1: rpc error: code = Unknown desc = out of range. Count must be within range [0,Capacity]. Found Count: -1, Capacity: 50\n", + counterName: "games", + wantCount: "COUNTER: 1\n", + }, + "GetCounterCapacity": { + msg: "GET_COUNTER_CAPACITY games", + want: "CAPACITY: 50\n", + }, + "GetCounterCapacity Counter Does Not Exist": { + msg: "GET_COUNTER_CAPACITY dame", + want: "ERROR: -1\n", + }, + "SetCounterCapacity": { + msg: "SET_COUNTER_CAPACITY qux 0", + want: "SUCCESS\n", + counterName: "qux", + wantCapacity: "CAPACITY: 0\n", + }, + "SetCounterCapacity Past Zero": { + msg: "SET_COUNTER_CAPACITY games -42", + want: "ERROR: could not set Counter games capacity to amount -42: rpc error: code = Unknown desc = out of range. Capacity must be greater than or equal to 0. Found Capacity: -42\n", + counterName: "games", + wantCount: "COUNTER: 1\n", + }, + } + // nolint:dupl // Linter errors on lines are duplicate of TestLists + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + logrus.WithField("msg", testCase.msg).Info(name) + reply, err := framework.SendGameServerUDP(t, gs, testCase.msg) + require.NoError(t, err) + assert.Equal(t, testCase.want, reply) + + if testCase.wantCount != "" { + msg := "GET_COUNTER_COUNT " + testCase.counterName + logrus.WithField("msg", msg).Info("Sending GetCounterCount") + reply, err = framework.SendGameServerUDP(t, gs, msg) + require.NoError(t, err) + assert.Equal(t, testCase.wantCount, reply) + } + + if testCase.wantCapacity != "" { + msg := "GET_COUNTER_CAPACITY " + testCase.counterName + logrus.WithField("msg", msg).Info("Sending GetCounterCapacity") + reply, err = framework.SendGameServerUDP(t, gs, msg) + require.NoError(t, err) + assert.Equal(t, testCase.wantCapacity, reply) + } + }) + } +} + +func TestLists(t *testing.T) { + if !runtime.FeatureEnabled(runtime.FeatureCountsAndLists) { + t.SkipNow() + } + t.Parallel() + ctx := context.Background() + gs := framework.DefaultGameServer(framework.Namespace) + gs.Labels = map[string]string{agonesv1.FleetNameLabel: "fleet-example"} + gs.Spec.Lists = make(map[string]agonesv1.ListStatus) + gs.Spec.Lists["players"] = agonesv1.ListStatus{ + Capacity: 1000, + } + gs.Spec.Lists["games"] = agonesv1.ListStatus{ + Values: []string{"game1", "game2"}, + Capacity: 50, + } + gs.Spec.Lists["foo"] = agonesv1.ListStatus{ + Values: []string{}, + Capacity: 1, + } + gs.Spec.Lists["bar"] = agonesv1.ListStatus{ + Values: []string{"bar1", "bar2"}, + Capacity: 10, + } + gs.Spec.Lists["baz"] = agonesv1.ListStatus{ + Values: []string{"baz1"}, + Capacity: 1, + } + gs.Spec.Lists["qux"] = agonesv1.ListStatus{ + Values: []string{"qux1", "qux2", "qux3", "qux4"}, + Capacity: 5, + } + + gs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + require.NoError(t, err) + defer framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, gs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck + assert.Equal(t, agonesv1.GameServerStateReady, gs.Status.State) + + testCases := map[string]struct { + msg string + want string + listName string + wantLength string + wantCapacity string + }{ + "GetListCapacity": { + msg: "GET_LIST_CAPACITY games", + want: "CAPACITY: 50\n", + }, + "SetListCapacity": { + msg: "SET_LIST_CAPACITY foo 1000", + want: "SUCCESS\n", + listName: "foo", + wantCapacity: "CAPACITY: 1000\n", + }, + "SetListCapacity past 1000": { + msg: "SET_LIST_CAPACITY games 1001", + want: "ERROR: could not set List games capacity to amount 1001: rpc error: code = Unknown desc = out of range. Capacity must be within range [0,1000]. Found Capacity: 1001\n", + listName: "games", + wantCapacity: "CAPACITY: 50\n", + }, + "SetListCapacity negative": { + msg: "SET_LIST_CAPACITY games -1", + want: "ERROR: could not set List games capacity to amount -1: rpc error: code = Unknown desc = out of range. Capacity must be within range [0,1000]. Found Capacity: -1\n", + listName: "games", + wantCapacity: "CAPACITY: 50\n", + }, + "ListContains": { + msg: "LIST_CONTAINS games game2", + want: "FOUND: true\n", + }, + "ListContains false": { + msg: "LIST_CONTAINS games game0", + want: "FOUND: false\n", + }, + "GetListLength": { + msg: "GET_LIST_LENGTH games", + want: "LENGTH: 2\n", + }, + "GetListValues": { + msg: "GET_LIST_VALUES games", + want: "VALUES: game1,game2\n", + }, + "GetListValues empty": { + msg: "GET_LIST_VALUES foo", + want: "VALUES: \n", + }, + "AppendListValue": { + msg: "APPEND_LIST_VALUE bar bar3", + want: "SUCCESS\n", + listName: "bar", + wantLength: "LENGTH: 3\n", + }, + "AppendListValue past capacity": { + msg: "APPEND_LIST_VALUE baz baz2", + want: "ERROR: could not get List baz: rpc error: code = Unknown desc = out of range. No available capacity. Current Capacity: 1, List Size: 1\n", + listName: "baz", + wantLength: "LENGTH: 1\n", + }, + "DeleteListValue": { + msg: "DELETE_LIST_VALUE qux qux3", + want: "SUCCESS\n", + listName: "qux", + wantLength: "LENGTH: 3\n", + }, + "DeleteListValue value does not exist": { + msg: "DELETE_LIST_VALUE games game4", + want: "ERROR: could not get List games: rpc error: code = Unknown desc = not found: value game4 not in list games\n", + listName: "games", + wantLength: "LENGTH: 2\n", + }, + } + // nolint:dupl // Linter errors on lines are duplicate of TestCounters + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + logrus.WithField("msg", testCase.msg).Info(name) + reply, err := framework.SendGameServerUDP(t, gs, testCase.msg) + require.NoError(t, err) + assert.Equal(t, testCase.want, reply) + + if testCase.wantLength != "" { + msg := "GET_LIST_LENGTH " + testCase.listName + logrus.WithField("msg", msg).Info("Sending GetListLength") + reply, err = framework.SendGameServerUDP(t, gs, msg) + require.NoError(t, err) + assert.Equal(t, testCase.wantLength, reply) + } + + if testCase.wantCapacity != "" { + msg := "GET_LIST_CAPACITY " + testCase.listName + logrus.WithField("msg", msg).Info("Sending GetListCapacity") + reply, err = framework.SendGameServerUDP(t, gs, msg) + require.NoError(t, err) + assert.Equal(t, testCase.wantCapacity, reply) + } + }) + } +} + +func TestSideCarCommunicatesWhileTerminating(t *testing.T) { + t.Parallel() + ctx := context.Background() + gs := framework.DefaultGameServer(framework.Namespace) + + minute := int64(60) + gs.Spec.Template.Spec.Containers[0].Args = append(gs.Spec.Template.Spec.Containers[0].Args, "--gracefulTerminationDelaySec", "60") + gs.Spec.Template.Spec.TerminationGracePeriodSeconds = &minute + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + require.NoError(t, err) + require.Equal(t, readyGs.Status.State, agonesv1.GameServerStateReady) + + // delete the GameServer + gameServers := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace) + err = gameServers.Delete(context.Background(), readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) + require.NoError(t, err, "Could not delete GameServer") + + // wait for the deletion timestamp to be set + err = wait.PollUntilContextTimeout(ctx, time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { + gs, err := gameServers.Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + return gs.DeletionTimestamp != nil, nil + }) + require.NoError(t, err, "Could not get a GameServer with deletion timestamp") + + // send a "GAMESERVER" message, and confirm it works + reply, err := framework.SendGameServerUDP(t, readyGs, "GAMESERVER") + require.NoError(t, err) + require.Equal(t, fmt.Sprintf("NAME: %s\n", readyGs.ObjectMeta.Name), reply) +} + +func TestGracefulShutdown(t *testing.T) { + // with the new sidecar pattern, there's no need for waiting on the Shutdown state. + if runtime.FeatureEnabled(runtime.FeatureSidecarContainers) { + t.SkipNow() + } + + t.Parallel() + + log := e2eframework.TestLogger(t) + ctx := context.Background() + gs := framework.DefaultGameServer(framework.Namespace) + var minute int64 = 60 + gs.Spec.Template.Spec.TerminationGracePeriodSeconds = &minute + readyGs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + if err != nil { + t.Fatalf("Could not get a GameServer ready: %v", err) + } + assert.Equal(t, readyGs.Status.State, agonesv1.GameServerStateReady) + gameservers := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace) + err = gameservers.Delete(ctx, readyGs.ObjectMeta.Name, metav1.DeleteOptions{}) + require.NoError(t, err) + log.Info("Deleted GameServer, waiting 20 seconds...") + time.Sleep(20 * time.Second) + log.WithField("gs", gs).Info("Checking GameServer") + gs, err = gameservers.Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, readyGs.ObjectMeta.Name, gs.ObjectMeta.Name) + + // move it to shutdown + gsCopy := gs.DeepCopy() + gsCopy.Status.State = agonesv1.GameServerStateShutdown + _, err = gameservers.Update(ctx, gsCopy, metav1.UpdateOptions{}) + require.NoError(t, err) + + start := time.Now() + require.Eventually(t, func() bool { + _, err := gameservers.Get(ctx, readyGs.ObjectMeta.Name, metav1.GetOptions{}) + log.WithError(err).Info("checking GameServer") + if err == nil { + return false + } + return k8serrors.IsNotFound(err) + }, 40*time.Second, time.Second) + + diff := int(time.Since(start).Seconds()) + log.WithField("diff", diff).Info("Time difference") + require.Less(t, diff, 40) +} + +func TestGameServerSlowStart(t *testing.T) { + t.Parallel() + + // Inject an additional game server sidecar that forces a delayed start + // to the main game server container following the pattern at + // https://medium.com/@marko.luksa/delaying-application-start-until-sidecar-is-ready-2ec2d21a7b74 + gs := framework.DefaultGameServer(framework.Namespace) + gs.Spec.Template.Spec.Containers = append( + []corev1.Container{{ + Name: "delay-game-server-start", + Image: "alpine:latest", + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"sleep", "3600"}, + Lifecycle: &corev1.Lifecycle{ + PostStart: &corev1.LifecycleHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"sleep", "60"}, + }, + }, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("30m"), + corev1.ResourceMemory: resource.MustParse("64Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("30m"), + corev1.ResourceMemory: resource.MustParse("64Mi"), + }, + }, + }}, + gs.Spec.Template.Spec.Containers...) + + // Validate that a game server whose primary container starts slowly (a full minute + // after the SDK starts) is capable of reaching Ready. Here we force the condition + // with a lifecycle hook, but it imitates a slow image pull, or other container + // start delays. + _, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + assert.NoError(t, err) +} + +func TestGameServerPatch(t *testing.T) { + if !runtime.FeatureEnabled(runtime.FeatureCountsAndLists) { + t.SkipNow() + } + t.Parallel() + ctx := context.Background() + + gs := framework.DefaultGameServer(framework.Namespace) + gs, err := framework.CreateGameServerAndWaitUntilReady(t, framework.Namespace, gs) + require.NoError(t, err) + defer framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Delete(ctx, gs.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint: errcheck + assert.Equal(t, agonesv1.GameServerStateReady, gs.Status.State) + + // Create a gameserver to patch against + gsCopy := gs.DeepCopy() + gsCopy.ObjectMeta.Labels = map[string]string{"foo": "foo-value"} + + patch, err := gs.Patch(gsCopy) + require.NoError(t, err) + patchString := string(patch) + require.Contains(t, patchString, fmt.Sprintf("{\"op\":\"test\",\"path\":\"/metadata/resourceVersion\",\"value\":%q}", gs.ObjectMeta.ResourceVersion)) + require.Contains(t, patchString, "{\"op\":\"add\",\"path\":\"/metadata/labels\",\"value\":{\"foo\":\"foo-value\"}}") + + // Confirm patch is applied correctly + patchedGs, err := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Patch(ctx, gs.GetObjectMeta().GetName(), types.JSONPatchType, patch, metav1.PatchOptions{}) + require.NoError(t, err) + require.Equal(t, patchedGs.ObjectMeta.Labels, map[string]string{"foo": "foo-value"}) + require.NotEqual(t, patchedGs.ObjectMeta.ResourceVersion, gs.ObjectMeta.ResourceVersion) + + // Confirm a patch applied to an old version of a game server is not applied + gsCopy.ObjectMeta.Labels = map[string]string{"bar": "bar-value"} + patch, err = gs.Patch(gsCopy) + require.NoError(t, err) + + _, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Patch(ctx, gs.GetObjectMeta().GetName(), types.JSONPatchType, patch, metav1.PatchOptions{}) + require.Error(t, err) + + getGs, err := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, gs.ObjectMeta.Name, metav1.GetOptions{}) + require.NoError(t, err) + require.Equal(t, getGs.ObjectMeta.Labels, map[string]string{"foo": "foo-value"}) + require.Equal(t, getGs.ObjectMeta.ResourceVersion, patchedGs.ObjectMeta.ResourceVersion) + + // Confirm patch goes through with the most up-to-date game server + gsCopy = patchedGs.DeepCopy() + gsCopy.ObjectMeta.Labels = map[string]string{"bar": "bar-value"} + patch, err = patchedGs.Patch(gsCopy) + require.NoError(t, err) + + rePatchedGs, err := framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Patch(ctx, gs.GetObjectMeta().GetName(), types.JSONPatchType, patch, metav1.PatchOptions{}) + require.NoError(t, err) + require.Equal(t, rePatchedGs.ObjectMeta.Labels, map[string]string{"bar": "bar-value"}) + + getGs, err = framework.AgonesClient.AgonesV1().GameServers(framework.Namespace).Get(ctx, gs.ObjectMeta.Name, metav1.GetOptions{}) + require.NoError(t, err) + require.Equal(t, getGs.ObjectMeta.Labels, map[string]string{"bar": "bar-value"}) + require.Equal(t, getGs.ObjectMeta.ResourceVersion, rePatchedGs.ObjectMeta.ResourceVersion) +} diff --git a/test/upgrade/versionMap.yaml b/test/upgrade/versionMap.yaml index 763663bd26..6c29598ba5 100644 --- a/test/upgrade/versionMap.yaml +++ b/test/upgrade/versionMap.yaml @@ -127,9 +127,15 @@ data: "alphaGates": ["PlayerAllocationFilter", "PlayerTracking", "WasmAutoscaler"], "betaGates": ["CountsAndLists", "GKEAutopilotExtendedDurationPods", "PortPolicyNone", "PortRanges", "RollingUpdateFix", "ScheduledAutoscaler", "FleetAutoscaleRequestMetaData", "SidecarContainers"] }, + "1.59.0": { + "alphaGates": ["PlayerAllocationFilter", "PlayerTracking", "WasmAutoscaler"], + "betaGates": ["CountsAndLists", "GKEAutopilotExtendedDurationPods", "PortRanges", "RollingUpdateFix", "ScheduledAutoscaler", "FleetAutoscaleRequestMetaData", "SidecarContainers"], + "stableGates": ["PortPolicyNone"] + }, "Dev": { "alphaGates": ["PlayerAllocationFilter", "PlayerTracking", "WasmAutoscaler"], - "betaGates": ["CountsAndLists", "GKEAutopilotExtendedDurationPods", "PortPolicyNone", "PortRanges", "RollingUpdateFix", "ScheduledAutoscaler", "FleetAutoscaleRequestMetaData", "SidecarContainers"] + "betaGates": ["CountsAndLists", "GKEAutopilotExtendedDurationPods", "PortRanges", "RollingUpdateFix", "ScheduledAutoscaler", "FleetAutoscaleRequestMetaData", "SidecarContainers"], + "stableGates": ["PortPolicyNone"] } } }