diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e06a4908f..11033989d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,7 +65,7 @@ jobs: run: make verify-codegen - name: Lint - run: make lint + run: make lint-changed - name: Build run: make build diff --git a/Makefile b/Makefile index 79e7c5567..2a6cadbe8 100644 --- a/Makefile +++ b/Makefile @@ -81,6 +81,11 @@ test-integration: ## Run integration tests. lint: ## Run golangci-lint. golangci-lint run ./... +.PHONY: lint-changed +lint-changed: REVISION ?= HEAD~1 +lint-changed: ## Run golangci-lint on changed files only. + golangci-lint run --new-from-rev=$(REVISION) ./... + .PHONY: clean clean: ## Remove generated artifacts. @echo "Cleaning generated artifacts..." diff --git a/api/device/v1alpha1/register.go b/api/device/v1alpha1/register.go index c6f6ea098..766382eaa 100644 --- a/api/device/v1alpha1/register.go +++ b/api/device/v1alpha1/register.go @@ -21,6 +21,7 @@ import ( ) var SchemeGroupVersion = schema.GroupVersion{Group: "device.nvidia.com", Version: "v1alpha1"} +var internalSchemeGroupVersion = schema.GroupVersion{Group: "device.nvidia.com", Version: runtime.APIVersionInternal} func Resource(resource string) schema.GroupResource { return SchemeGroupVersion.WithResource(resource).GroupResource() @@ -40,6 +41,11 @@ func addKnownTypes(scheme *runtime.Scheme) error { &GPUList{}, ) + scheme.AddKnownTypes(internalSchemeGroupVersion, + &GPU{}, + &GPUList{}, + ) + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil diff --git a/cmd/device-apiserver/app/server_test.go b/cmd/device-apiserver/app/server_test.go index a81dac2da..2adce9d30 100644 --- a/cmd/device-apiserver/app/server_test.go +++ b/cmd/device-apiserver/app/server_test.go @@ -31,7 +31,7 @@ func TestRun(t *testing.T) { localSocket := testutils.NewUnixAddr(t) kineSocket := fmt.Sprintf("unix://%s", testutils.NewUnixAddr(t)) - healthAddr := testutils.GetFreeTCPAddress(t) + healthAddr := testutils.MustGetFreeTCPAddress(t) opts.GRPC.BindAddress = "unix://" + localSocket opts.HealthAddress = healthAddr @@ -90,7 +90,7 @@ func TestRun_StorageFailure(t *testing.T) { opts.Storage.KineSocketPath = filepath.Join(readOnlyDir, "kine.sock") opts.Storage.KineConfig.Endpoint = fmt.Sprintf("sqlite://%s/db.sqlite", readOnlyDir) - opts.HealthAddress = testutils.GetFreeTCPAddress(t) + opts.HealthAddress = testutils.MustGetFreeTCPAddress(t) opts.GRPC.BindAddress = "unix://" + filepath.Join(tmpDir, "api.sock") ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) diff --git a/pkg/controlplane/apiserver/server_test.go b/pkg/controlplane/apiserver/server_test.go index 6ad126574..008013563 100644 --- a/pkg/controlplane/apiserver/server_test.go +++ b/pkg/controlplane/apiserver/server_test.go @@ -34,8 +34,8 @@ func TestDeviceAPIServer(t *testing.T) { dbPath := filepath.Join(tmpDir, "test.db") kineSocket := testutils.NewUnixAddr(t) apiSocket := testutils.NewUnixAddr(t) - healthAddr := testutils.GetFreeTCPAddress(t) - metricsAddr := testutils.GetFreeTCPAddress(t) + healthAddr := testutils.MustGetFreeTCPAddress(t) + metricsAddr := testutils.MustGetFreeTCPAddress(t) storage := &storagebackend.Storage{ KineSocketPath: "unix://" + kineSocket, diff --git a/pkg/services/device/v1alpha1/gpu_provider.go b/pkg/services/device/v1alpha1/gpu_provider.go index 32dc779bd..fd38498b5 100644 --- a/pkg/services/device/v1alpha1/gpu_provider.go +++ b/pkg/services/device/v1alpha1/gpu_provider.go @@ -47,15 +47,24 @@ func NewGPUServiceProvider() api.ServiceProvider { } func (p *gpuServiceProvider) Install(svr *grpc.Server, storageConfig storagebackend.Config) (api.Service, error) { - gv := p.groupVersion.String() - scheme := runtime.NewScheme() if err := devicev1alpha1.AddToScheme(scheme); err != nil { - return nil, fmt.Errorf("failed to add %q to scheme: %w", gv, err) + return nil, fmt.Errorf("failed to add %q to scheme: %w", p.groupVersion.String(), err) } - + codecs := serializer.NewCodecFactory(scheme) - codec := codecs.LegacyCodec(p.groupVersion) + + info, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), runtime.ContentTypeJSON) + if !ok { + return nil, fmt.Errorf("unable to find serializer for JSON") + } + + codec := codecs.CodecForVersions( + info.Serializer, + codecs.UniversalDecoder(p.groupVersion), + p.groupVersion, + runtime.InternalGroupVersioner, + ) configForResource := storagebackend.ConfigForResource{ Config: storageConfig, @@ -75,7 +84,6 @@ func (p *gpuServiceProvider) Install(svr *grpc.Server, storageConfig storageback } service := NewGPUService(s, destroyFunc) - pb.RegisterGpuServiceServer(svr, service) return service, nil diff --git a/pkg/services/device/v1alpha1/gpu_service.go b/pkg/services/device/v1alpha1/gpu_service.go index 3bff930d5..e129bf5dd 100644 --- a/pkg/services/device/v1alpha1/gpu_service.go +++ b/pkg/services/device/v1alpha1/gpu_service.go @@ -85,21 +85,28 @@ func (s *gpuService) GetGpu(ctx context.Context, req *pb.GetGpuRequest) (*pb.Get return nil, status.Error(codes.InvalidArgument, "name is required") } - key := s.storageKey(req.GetNamespace(), req.GetName()) + name := req.GetName() + ns := req.GetNamespace() + reqRV := "" + if req.GetOpts() != nil { + reqRV = req.GetOpts().GetResourceVersion() + } + + key := s.storageKey(ns, name) opts := storage.GetOptions{ - ResourceVersion: req.GetOpts().GetResourceVersion(), + ResourceVersion: reqRV, } gpu := &devicev1alpha1.GPU{} if err := s.storage.Get(ctx, key, opts, gpu); err != nil { if storage.IsNotFound(err) { - return nil, status.Errorf(codes.NotFound, "GPU %q not found", req.GetName()) + return nil, status.Errorf(codes.NotFound, "GPU %q not found", name) } logger.V(3).Error(err, "storage backend error during Get", "key", key) return nil, status.Error(codes.Internal, "internal server error") } - logger.V(4).Info("Retrieved GPU", "name", req.GetName(), "namespace", req.GetNamespace()) + logger.V(4).Info("Retrieved GPU", "name", name, "namespace", ns) return &pb.GetGpuResponse{ Gpu: devicev1alpha1.ToProto(gpu), @@ -109,27 +116,27 @@ func (s *gpuService) GetGpu(ctx context.Context, req *pb.GetGpuRequest) (*pb.Get func (s *gpuService) ListGpus(ctx context.Context, req *pb.ListGpusRequest) (*pb.ListGpusResponse, error) { logger := klog.FromContext(ctx) - var gpus devicev1alpha1.GPUList + ns := req.GetNamespace() + reqRV := "" + if req.GetOpts() != nil { + reqRV = req.GetOpts().GetResourceVersion() + } + key := s.storageKey(ns, "") opts := storage.ListOptions{ - ResourceVersion: req.GetOpts().GetResourceVersion(), + ResourceVersion: reqRV, Recursive: true, Predicate: storage.Everything, // TODO: selection predicate } - key := s.storageKey(req.GetNamespace(), "") + var gpus devicev1alpha1.GPUList if err := s.storage.GetList(ctx, key, opts, &gpus); err != nil { if storage.IsNotFound(err) { - rv, _ := s.storage.GetCurrentResourceVersion(ctx) - rvStr := fmt.Sprintf("%d", rv) - if rv == 0 { - rvStr = req.GetOpts().GetResourceVersion() - } - + currRV, _ := s.storage.GetCurrentResourceVersion(ctx) return &pb.ListGpusResponse{ GpuList: &pb.GpuList{ Metadata: &pb.ListMeta{ - ResourceVersion: rvStr, + ResourceVersion: fmt.Sprintf("%d", currRV), }, Items: []*pb.Gpu{}, }, @@ -140,7 +147,7 @@ func (s *gpuService) ListGpus(ctx context.Context, req *pb.ListGpusRequest) (*pb } logger.V(4).Info("Listed GPUs", - "namespace", req.GetNamespace(), + "namespace", ns, "count", len(gpus.Items), "resourceVersion", gpus.GetListMeta().GetResourceVersion(), ) @@ -155,14 +162,19 @@ func (s *gpuService) WatchGpus(req *pb.WatchGpusRequest, stream pb.GpuService_Wa logger := klog.FromContext(ctx) ns := req.GetNamespace() - rv := req.GetOpts().GetResourceVersion() + reqRV := "" + if req.GetOpts() != nil { + reqRV = req.GetOpts().GetResourceVersion() + } - key := s.storageKey(req.GetNamespace(), "") - w, err := s.storage.Watch(ctx, key, storage.ListOptions{ - ResourceVersion: req.GetOpts().GetResourceVersion(), + key := s.storageKey(ns, "") + opts := storage.ListOptions{ + ResourceVersion: reqRV, Recursive: true, Predicate: storage.Everything, // TODO: selection predicate - }) + } + + w, err := s.storage.Watch(ctx, key, opts) if err != nil { if storage.IsInvalidError(err) { return status.Errorf(codes.OutOfRange, "%v", err) @@ -172,7 +184,7 @@ func (s *gpuService) WatchGpus(req *pb.WatchGpusRequest, stream pb.GpuService_Wa } defer w.Stop() - logger.V(3).Info("Started watch stream", "namespace", ns, "resourceVersion", rv) + logger.V(3).Info("Started watch stream", "namespace", ns, "resourceVersion", reqRV) for { select { @@ -188,8 +200,8 @@ func (s *gpuService) WatchGpus(req *pb.WatchGpusRequest, stream pb.GpuService_Wa if event.Type == watch.Error { if statusObj, ok := event.Object.(*metav1.Status); ok { if statusObj.Code == 410 || statusObj.Reason == metav1.StatusReasonExpired { - logger.V(4).Info("Watch stream expired", "namespace", ns, "resourceVersion", rv) - return status.Errorf(codes.OutOfRange, "required resource version %s is too old: %s", rv, statusObj.Message) + logger.V(4).Info("Watch stream expired", "namespace", ns, "resourceVersion", reqRV) + return status.Errorf(codes.OutOfRange, "required resource version %s is too old: %s", reqRV, statusObj.Message) } logger.Error(nil, "watch stream storage status error", "status", statusObj.Message) return status.Error(codes.Internal, "internal server error") @@ -243,24 +255,25 @@ func (s *gpuService) CreateGpu(ctx context.Context, req *pb.CreateGpuRequest) (* } key := s.storageKey(ns, name) - gpu := devicev1alpha1.FromProto(req.Gpu) - // TODO: move into PrepareForCreate function? - gpu.SetNamespace(ns) - gpu.SetUID(uuid.NewUUID()) + in := devicev1alpha1.FromProto(req.Gpu) + in.SetNamespace(ns) + in.SetUID(uuid.NewUUID()) + now := metav1.Now() - gpu.SetCreationTimestamp(now) - gpu.SetGeneration(1) - for i := range gpu.Status.Conditions { - if gpu.Status.Conditions[i].LastTransitionTime.IsZero() { - gpu.Status.Conditions[i].LastTransitionTime = now + in.SetCreationTimestamp(now) + in.SetGeneration(1) + + for i := range in.Status.Conditions { + if in.Status.Conditions[i].LastTransitionTime.IsZero() { + in.Status.Conditions[i].LastTransitionTime = now } } out := &devicev1alpha1.GPU{} - if err := s.storage.Create(ctx, key, gpu, out, 0); err != nil { + if err := s.storage.Create(ctx, key, in, out, 0); err != nil { logger.Error(err, "Failed to create GPU", "name", name, "namespace", ns) if storage.IsExist(err) { - return nil, status.Errorf(codes.AlreadyExists, "GPU %q already exists", req.GetGpu().GetMetadata().GetName()) + return nil, status.Errorf(codes.AlreadyExists, "GPU %q already exists", name) } return nil, status.Error(codes.Internal, "internal server error") } @@ -283,12 +296,12 @@ func (s *gpuService) UpdateGpu(ctx context.Context, req *pb.UpdateGpuRequest) (* name := req.GetGpu().GetMetadata().GetName() ns := req.GetGpu().GetMetadata().GetNamespace() key := s.storageKey(ns, name) - updatedGpu := &devicev1alpha1.GPU{} + updated := &devicev1alpha1.GPU{} err := s.storage.GuaranteedUpdate( ctx, key, - updatedGpu, + updated, false, // ignoreNotFound nil, // TODO: preconditions func(input runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) { @@ -324,7 +337,7 @@ func (s *gpuService) UpdateGpu(ctx context.Context, req *pb.UpdateGpuRequest) (* if err != nil { if storage.IsNotFound(err) { - return nil, status.Errorf(codes.NotFound, "GPU %q not found", req.GetGpu().GetMetadata().GetName()) + return nil, status.Errorf(codes.NotFound, "GPU %q not found", name) } if storage.IsConflict(err) { logger.V(3).Info("Update conflict", "name", name, "namespace", ns, "err", err) @@ -338,11 +351,11 @@ func (s *gpuService) UpdateGpu(ctx context.Context, req *pb.UpdateGpuRequest) (* logger.V(2).Info("Successfully updated GPU", "name", name, "namespace", ns, - "resourceVersion", updatedGpu.ResourceVersion, - "generation", updatedGpu.Generation, + "resourceVersion", updated.ResourceVersion, + "generation", updated.Generation, ) - return devicev1alpha1.ToProto(updatedGpu), nil + return devicev1alpha1.ToProto(updated), nil } func (s *gpuService) DeleteGpu(ctx context.Context, req *pb.DeleteGpuRequest) (*emptypb.Empty, error) { @@ -354,9 +367,12 @@ func (s *gpuService) DeleteGpu(ctx context.Context, req *pb.DeleteGpuRequest) (* name := req.GetName() ns := req.GetNamespace() + if ns == "" { + ns = "default" + } key := s.storageKey(ns, name) - out := &devicev1alpha1.GPU{} + out := &devicev1alpha1.GPU{} if err := s.storage.Delete( ctx, key, diff --git a/pkg/util/testutils/net.go b/pkg/util/testutils/net.go index 6494a3ed7..192334bbd 100644 --- a/pkg/util/testutils/net.go +++ b/pkg/util/testutils/net.go @@ -29,43 +29,55 @@ import ( "k8s.io/apimachinery/pkg/util/wait" ) -// NewUnixAddr creates a temporary directory and returns a path for a UDS socket. -// Uses t.Cleanup to ensure the directory is removed when the test finishes. +// CreateUnixAddr creates a temporary directory and returns a socket path and the directory. +func CreateUnixAddr() (string, string, error) { + d, err := os.MkdirTemp("", "test-uds-") + if err != nil { + return "", "", err + } + + return filepath.Join(d, "api.sock"), d, nil +} + +// NewUnixAddr creates a temporary socket path and registers directory cleanup. func NewUnixAddr(t testing.TB) string { t.Helper() - d, err := os.MkdirTemp("", "test-uds-") + path, dir, err := CreateUnixAddr() if err != nil { - t.Fatalf("failed to create temp dir for socket: %v", err) + t.Fatalf("failed to create socket: %v", err) } t.Cleanup(func() { - if err := os.RemoveAll(d); err != nil { - t.Errorf("failed to cleanup temp socket dir %q: %v", d, err) - } + os.RemoveAll(dir) }) - return filepath.Join(d, "api.sock") + return path } // GetFreeTCPAddress finds an available port on the loopback interface. -func GetFreeTCPAddress(t *testing.T) string { - t.Helper() - +func GetFreeTCPAddress() (string, error) { lc := net.ListenConfig{} lis, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") if err != nil { - t.Fatalf("failed to find a free port: %v", err) + return "", err } + defer lis.Close() - address := lis.Addr().String() + return lis.Addr().String(), nil +} - if err := lis.Close(); err != nil { - t.Fatalf("failed to close temporary listener: %v", err) +// MustGetFreeTCPAddress finds an available port or fails the test. +func MustGetFreeTCPAddress(t *testing.T) string { + t.Helper() + + addr, err := GetFreeTCPAddress() + if err != nil { + t.Fatalf("failed to find a free port: %v", err) } - return address + return addr } type HealthCondition func(resp *healthpb.HealthCheckResponse) bool @@ -80,9 +92,8 @@ var ( } ) -func WaitForStatus(t *testing.T, addr string, serviceName string, timeout time.Duration, condition HealthCondition) { - t.Helper() - +// PollHealthStatus polls the health service until the condition is met or the timeout is reached. +func PollHealthStatus(addr string, serviceName string, timeout time.Duration, condition HealthCondition) error { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -93,13 +104,13 @@ func WaitForStatus(t *testing.T, addr string, serviceName string, timeout time.D grpc.WithTransportCredentials(insecure.NewCredentials()), ) if err != nil { - t.Fatalf("Failed to create client for %s: %v", addr, err) + return fmt.Errorf("failed to create client for %s: %w", addr, err) } defer conn.Close() client := healthpb.NewHealthClient(conn) - err = wait.PollUntilContextTimeout(ctx, 200*time.Millisecond, timeout, true, func(ctx context.Context) (bool, error) { + return wait.PollUntilContextTimeout(ctx, 200*time.Millisecond, timeout, true, func(ctx context.Context) (bool, error) { callCtx, callCancel := context.WithTimeout(ctx, 500*time.Millisecond) defer callCancel() @@ -110,7 +121,13 @@ func WaitForStatus(t *testing.T, addr string, serviceName string, timeout time.D return condition(resp), nil }) - if err != nil { +} + +// WaitForStatus waits for a health condition to be met or fails the test. +func WaitForStatus(t *testing.T, addr string, serviceName string, timeout time.Duration, condition HealthCondition) { + t.Helper() + + if err := PollHealthStatus(addr, serviceName, timeout, condition); err != nil { t.Fatalf("Condition for %s not met on %s within %v: %v", serviceName, addr, timeout, err) } } diff --git a/test/integration/client-go/device/v1alpha1/clientset_test.go b/test/integration/client-go/device/v1alpha1/clientset_test.go deleted file mode 100644 index 6745e3003..000000000 --- a/test/integration/client-go/device/v1alpha1/clientset_test.go +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package v1alpha1_test - -import ( - "context" - "encoding/json" - "fmt" - "strconv" - "testing" - "time" - - devicev1alpha1 "github.com/nvidia/nvsentinel/api/device/v1alpha1" - "github.com/nvidia/nvsentinel/cmd/device-apiserver/app" - "github.com/nvidia/nvsentinel/cmd/device-apiserver/app/options" - "github.com/nvidia/nvsentinel/pkg/client-go/client/versioned" - "github.com/nvidia/nvsentinel/pkg/grpc/client" - "github.com/nvidia/nvsentinel/pkg/util/testutils" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestEndToEnd(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - tmpDir := t.TempDir() - - socketPath := testutils.NewUnixAddr(t) - kineSocket := fmt.Sprintf("unix://%s", testutils.NewUnixAddr(t)) - healthAddr := testutils.GetFreeTCPAddress(t) - - opts := options.NewServerRunOptions() - opts.NodeName = "test-node" - opts.GRPC.BindAddress = "unix://" + socketPath - opts.HealthAddress = healthAddr - opts.Storage.DatabaseDir = tmpDir - opts.Storage.DatabasePath = tmpDir + "state.db" - opts.Storage.KineSocketPath = kineSocket - opts.Storage.KineConfig.Endpoint = fmt.Sprintf("sqlite://%s/db.sqlite", tmpDir) - opts.Storage.KineConfig.Listener = kineSocket - - completed, err := opts.Complete(ctx) - if err != nil { - t.Fatalf("Failed to complete options: %v", err) - } - - go func() { - if err := app.Run(ctx, completed); err != nil && err != context.Canceled { - t.Errorf("Server exited with error: %v", err) - } - }() - - testutils.WaitForStatus(t, healthAddr, "", 5*time.Second, testutils.IsServing) - - config := &client.Config{Target: "unix://" + socketPath} - client, err := versioned.NewForConfig(config) - if err != nil { - t.Fatalf("Failed to create clientset: %v", err) - } - - var created *devicev1alpha1.GPU - - t.Run("Create", func(t *testing.T) { - gpu := &devicev1alpha1.GPU{ - ObjectMeta: metav1.ObjectMeta{ - Name: "gpu-ad2367dd-a40e-6b86-6fc3-c44a2cc92c7e", - }, - Spec: devicev1alpha1.GPUSpec{ - UUID: "GPU-ad2367dd-a40e-6b86-6fc3-c44a2cc92c7e", - }, - Status: devicev1alpha1.GPUStatus{ - Conditions: []metav1.Condition{ - { - Type: "Ready", - Status: metav1.ConditionFalse, - Reason: "DriverNotReaady", - Message: "Driver is posting ready status", - }, - }, - }, - } - - created, err = client.DeviceV1alpha1().GPUs().Create(ctx, gpu, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("Failed to create GPU: %v", err) - } - - // Client generated fields - if created.Kind != "GPU" { - t.Errorf("expected kind 'GPU', got %s", created.Kind) - } - if created.APIVersion != devicev1alpha1.SchemeGroupVersion.String() { - t.Errorf("expected version %s, got %s", devicev1alpha1.SchemeGroupVersion.String(), created.APIVersion) - } - - // Server generated fields - if created.Namespace != "default" { - t.Error("Server failed to set default namespace") - } - if created.UID == "" { - t.Error("Server failed to generate a UID for the GPU") - } - if created.ResourceVersion == "" { - t.Error("Server failed to generate a ResourceVersion") - } - if created.Generation != 1 { - t.Error("Server failed to set initial Generation") - } - if created.CreationTimestamp.IsZero() { - t.Error("Server failed to set a CreationTimestamp") - } - - // Data integrity - if created.Name != gpu.Name { - t.Errorf("expected name %q, got %q", gpu.Name, created.Name) - } - if created.Spec.UUID != gpu.Spec.UUID { - t.Errorf("expected UUID %q, got %q", gpu.Spec.UUID, created.Spec.UUID) - } - - // Data integrity: Status - if len(created.Status.Conditions) != len(gpu.Status.Conditions) { - t.Fatalf("expected %d conditions, got %d", len(gpu.Status.Conditions), len(created.Status.Conditions)) - } - - cond := created.Status.Conditions[0] - expected := gpu.Status.Conditions[0] - - if cond.Type != expected.Type { - t.Errorf("expected condition Type %q, got %q", expected.Type, cond.Type) - } - if cond.Status != expected.Status { - t.Errorf("expected condition Status %q, got %q", expected.Status, cond.Status) - } - if cond.Reason != expected.Reason { - t.Errorf("expected condition Reason %q, got %q", expected.Reason, cond.Reason) - } - if cond.Message != expected.Message { - t.Errorf("expected condition Message %q, got %q", expected.Message, cond.Message) - } - if cond.LastTransitionTime.IsZero() { - t.Error("condition LastTransitionTime should not be zero") - } - - // TODO: remove - objJson, _ := json.MarshalIndent(created, "", " ") - fmt.Printf("\n--- [Object After Creation] ---\n%s\n", string(objJson)) - }) - - t.Run("Update", func(t *testing.T) { - if created == nil { - t.Skip("Skipping: Create failed") - } - - toUpdate := created.DeepCopy() - toUpdate.Spec.UUID = "GPU-cd2367dd-a40e-6b86-6fc3-c44a2cc92c7d" - - updated, err := client.DeviceV1alpha1().GPUs().Update(ctx, toUpdate, metav1.UpdateOptions{}) - if err != nil { - t.Fatalf("Failed to update GPU: %v", err) - } - - if updated.Spec.UUID != toUpdate.Spec.UUID { - t.Errorf("expected UUID %q, got %q", toUpdate.Spec.UUID, updated.Spec.UUID) - } - - oldRV, _ := strconv.ParseInt(created.ResourceVersion, 10, 64) - updatedRV, _ := strconv.ParseInt(updated.ResourceVersion, 10, 64) - - if updatedRV <= oldRV { - t.Errorf("expected ResourceVersion to increase, got %d (old) and %d (new)", oldRV, updatedRV) - } - - if updated.Generation <= created.Generation { - t.Errorf("expected Generation to increase, got %d (old) and %d (new)", created.Generation, updated.Generation) - } - - // TODO: remove - objJson, _ := json.MarshalIndent(updated, "", " ") - fmt.Printf("\n--- [Object After Update] ---\n%s\n", string(objJson)) - }) - - // TODO: add tests for Delete, List, Watch -} diff --git a/test/integration/client-go/device/v1alpha1/fake_client_test.go b/test/integration/client-go/device/v1alpha1/fake_client_test.go deleted file mode 100644 index afe72be48..000000000 --- a/test/integration/client-go/device/v1alpha1/fake_client_test.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package v1alpha1_test - -import ( - "context" - "testing" - "time" - - devicev1alpha1 "github.com/nvidia/nvsentinel/api/device/v1alpha1" - "github.com/nvidia/nvsentinel/pkg/client-go/client/versioned/fake" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/watch" -) - -func TestGPUFakeClient(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - gpu1 := &devicev1alpha1.GPU{ - ObjectMeta: metav1.ObjectMeta{ - Name: "gpu-1", - ResourceVersion: "100", - }, - Spec: devicev1alpha1.GPUSpec{UUID: "GPU-1"}, - } - client := fake.NewSimpleClientset(gpu1) - - gpu, err := client.DeviceV1alpha1().GPUs().Get(ctx, "gpu-1", metav1.GetOptions{}) - if err != nil { - t.Errorf("Fake client failed to retrieve GPU: %v", err) - } - if gpu.Name != "gpu-1" { - t.Errorf("Expected %v, got %v", gpu1, gpu) - } - if gpu.ResourceVersion != "100" { - t.Errorf("ResourceVersion mismatch: expected 100, got %s", gpu.ResourceVersion) - } - if gpu.Spec.UUID != "GPU-1" { - t.Errorf("UUID mismatch: expected GPU-1, got %s", gpu.Spec.UUID) - } - - list, err := client.DeviceV1alpha1().GPUs().List(ctx, metav1.ListOptions{}) - if err != nil { - t.Fatalf("Fake client failed to list GPUs: %v", err) - } - if len(list.Items) != 1 { - t.Errorf("Expected 1 GPU, got %d", len(list.Items)) - } - - watchRV := list.ResourceVersion - watcher, err := client.DeviceV1alpha1().GPUs().Watch(ctx, metav1.ListOptions{ - ResourceVersion: watchRV, - }) - if err != nil { - t.Fatalf("Fake client failed to Watch GPUs: %v", err) - } - defer watcher.Stop() - - // Simulate an Event in the background - gpu2 := &devicev1alpha1.GPU{ - ObjectMeta: metav1.ObjectMeta{ - Name: "gpu-2", - ResourceVersion: "101", - }, - Spec: devicev1alpha1.GPUSpec{UUID: "GPU-2"}, - } - go func() { - time.Sleep(100 * time.Millisecond) - client.Tracker().Add(gpu2) - }() - - select { - case event, ok := <-watcher.ResultChan(): - if !ok { - t.Fatal("Watch channel closed prematurely") - } - if event.Type != watch.Added { - t.Errorf("Expected Added event, got %v", event.Type) - } - obj := event.Object.(*devicev1alpha1.GPU) - if obj.Name != "gpu-2" { - t.Errorf("Expected gpu-2, got %s", obj.Name) - } - if obj.ResourceVersion != "101" { - t.Errorf("ResourceVersion mismatch: expected 101, got %s", obj.ResourceVersion) - } - case <-ctx.Done(): - t.Fatal("Timed out waiting for watch event") - } -} diff --git a/test/integration/device/v1alpha1/gpu_test.go b/test/integration/device/v1alpha1/gpu_test.go new file mode 100644 index 000000000..ddf27c0b2 --- /dev/null +++ b/test/integration/device/v1alpha1/gpu_test.go @@ -0,0 +1,321 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1_test + +import ( + "context" + "reflect" + "strconv" + "testing" + "time" + + devicev1alpha1 "github.com/nvidia/nvsentinel/api/device/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestGPU(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + client := clientset + + gpu1 := &devicev1alpha1.GPU{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gpu-11111111-1111-1111-1111-111111111111", + }, + Spec: devicev1alpha1.GPUSpec{ + UUID: "GPU-11111111-1111-1111-1111-111111111111", + }, + Status: devicev1alpha1.GPUStatus{ + Conditions: []metav1.Condition{ + { + Type: "Ready", + Status: metav1.ConditionFalse, + Reason: "DriverNotReaady", + Message: "Driver is posting ready status", + }, + }, + }, + } + gpu2 := &devicev1alpha1.GPU{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gpu-22222222-2222-2222-2222-222222222222", + }, + Spec: devicev1alpha1.GPUSpec{ + UUID: "GPU-22222222-2222-2222-2222-222222222222", + }, + Status: devicev1alpha1.GPUStatus{ + Conditions: []metav1.Condition{ + { + Type: "Ready", + Status: metav1.ConditionTrue, + Reason: "DriverReaady", + Message: "Driver is posting ready status", + }, + }, + }, + } + gpu3 := &devicev1alpha1.GPU{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gpu-33333333-3333-3333-3333-333333333333", + Namespace: "default", + }, + Spec: devicev1alpha1.GPUSpec{ + UUID: "GPU-33333333-3333-3333-3333-333333333333", + }, + Status: devicev1alpha1.GPUStatus{ + Conditions: []metav1.Condition{ + { + Type: "Ready", + Status: metav1.ConditionTrue, + Reason: "DriverReaady", + Message: "Driver is posting ready status", + }, + }, + }, + } + + var created1, created2, created3 *devicev1alpha1.GPU + var err error + + t.Run("Create", func(t *testing.T) { + created1, err = client.DeviceV1alpha1().GPUs().Create(ctx, gpu1, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create GPU: %v", err) + } + + // Client generated fields + if created1.Kind != "GPU" { + t.Errorf("expected kind 'GPU', got %s", created1.Kind) + } + if created1.APIVersion != devicev1alpha1.SchemeGroupVersion.String() { + t.Errorf("expected version %s, got %s", devicev1alpha1.SchemeGroupVersion.String(), created1.APIVersion) + } + + // Server generated fields + if created1.Namespace != "default" { + t.Error("server failed to set default namespace") + } + if created1.UID == "" { + t.Error("server failed to generate a UID for the GPU") + } + if created1.ResourceVersion == "" { + t.Error("server failed to generate a ResourceVersion") + } + if created1.Generation != 1 { + t.Error("server failed to set initial Generation") + } + if created1.CreationTimestamp.IsZero() { + t.Error("server failed to set a CreationTimestamp") + } + + // Data integrity + if created1.Name != gpu1.Name { + t.Errorf("expected name %q, got %q", gpu1.Name, created1.Name) + } + if created1.Spec.UUID != gpu1.Spec.UUID { + t.Errorf("expected UUID %q, got %q", gpu1.Spec.UUID, created1.Spec.UUID) + } + + // Data integrity: Status + if len(created1.Status.Conditions) != len(gpu1.Status.Conditions) { + t.Fatalf("Expected %d conditions, got %d", len(gpu1.Status.Conditions), len(created1.Status.Conditions)) + } + + cond := created1.Status.Conditions[0] + expected := gpu1.Status.Conditions[0] + + if cond.Type != expected.Type { + t.Errorf("expected condition Type %q, got %q", expected.Type, cond.Type) + } + if cond.Status != expected.Status { + t.Errorf("expected condition Status %q, got %q", expected.Status, cond.Status) + } + if cond.Reason != expected.Reason { + t.Errorf("expected condition Reason %q, got %q", expected.Reason, cond.Reason) + } + if cond.Message != expected.Message { + t.Errorf("expected condition Message %q, got %q", expected.Message, cond.Message) + } + if cond.LastTransitionTime.IsZero() { + t.Error("condition LastTransitionTime should not be zero") + } + }) + + t.Run("Update", func(t *testing.T) { + toUpdate := created1.DeepCopy() + toUpdate.Spec.UUID = "GPU-updated1-1111-1111-1111-111111111111" + + updated, err := client.DeviceV1alpha1().GPUs().Update(ctx, toUpdate, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("Failed to update GPU: %v", err) + } + + // Metadata + oldRV, _ := strconv.ParseInt(created1.ResourceVersion, 10, 64) + updatedRV, _ := strconv.ParseInt(updated.ResourceVersion, 10, 64) + + if updated.UID != created1.UID { + t.Errorf("expected UID to remain the same, got %v (old) and %v (new)", created1.UID, updated.UID) + } + if updated.Namespace != created1.Namespace { + t.Errorf("expected Namespace to remain the same, got %v (old) and %v (new)", created1.Namespace, updated.Namespace) + } + if updated.Name != created1.Name { + t.Errorf("expected Name to remain the same, got %v (old) and %v (new)", created1.Name, updated.Name) + } + if updatedRV <= oldRV { + t.Errorf("expected ResourceVersion to increase, got %d (old) and %d (new)", oldRV, updatedRV) + } + if updated.Generation <= created1.Generation { + t.Errorf("expected Generation to increase, got %d (old) and %d (new)", created1.Generation, updated.Generation) + } + if updated.CreationTimestamp != created1.CreationTimestamp { + t.Errorf("expected CreationTimestamp to remain the same, got %v (old) and %v (new)", created1.CreationTimestamp, updated.CreationTimestamp) + } + + // Spec + if updated.Spec.UUID != toUpdate.Spec.UUID { + t.Errorf("expected UUID %q, got %q", toUpdate.Spec.UUID, updated.Spec.UUID) + } + + // Status + if !reflect.DeepEqual(updated.Status, created1.Status) { + t.Errorf("Status changed during spec update!\nOld: %+v\nNew: %+v", + created1.Status, updated.Status) + } + }) + + t.Run("UpdateStatus", func(t *testing.T) { + t.Skip("Skipping: API not implemented") + }) + + t.Run("Patch", func(t *testing.T) { + t.Skip("Skipping: API not implemented") + }) + + t.Run("List", func(t *testing.T) { + created2, err = client.DeviceV1alpha1().GPUs().Create(ctx, gpu2, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create GPU: %v", err) + } + + list, err := client.DeviceV1alpha1().GPUs().List(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatalf("Failed to list GPUs: %v", err) + } + + if len(list.Items) != 2 { + t.Errorf("expected 2 GPUs in list, got %d", len(list.Items)) + } + + expectedNames := map[string]bool{ + created1.Name: false, + created2.Name: false, + } + for _, item := range list.Items { + if _, exists := expectedNames[item.Name]; exists { + expectedNames[item.Name] = true + } + } + for name, found := range expectedNames { + if !found { + t.Errorf("expected GPU %q was not found in the filtered list", name) + } + } + + lastObjectRV, _ := strconv.ParseUint(created2.ResourceVersion, 10, 64) + listRV, _ := strconv.ParseUint(list.ResourceVersion, 10, 64) + if listRV < lastObjectRV { + t.Errorf("ResourceVersion (%d) is behind the last object's RV (%d)", listRV, lastObjectRV) + } + + }) + + t.Run("ListAndWatch", func(t *testing.T) { + // Start watching from the RV of GPU 2 + opts := metav1.ListOptions{ + ResourceVersion: created2.ResourceVersion, + } + + watcher, err := client.DeviceV1alpha1().GPUs().Watch(ctx, opts) + if err != nil { + t.Fatalf("Failed to start watch: %v", err) + } + defer watcher.Stop() + + created3, err = client.DeviceV1alpha1().GPUs().Create(ctx, gpu3, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create third GPU: %v", err) + } + + select { + case event, ok := <-watcher.ResultChan(): + if !ok { + t.Fatal("Watch channel closed prematurely") + } + + gpu := event.Object.(*devicev1alpha1.GPU) + if gpu.Name != created3.Name { + t.Errorf("expected event for %s, got %s", created3.Name, gpu.Name) + } + if event.Type != "ADDED" { + t.Errorf("expected event type ADDED, got %s", event.Type) + } + + case <-time.After(5 * time.Second): + t.Fatal("Timed out waiting for Watch event for GPU 3") + } + }) + + t.Run("Delete", func(t *testing.T) { + err := client.DeviceV1alpha1().GPUs().Delete(ctx, created1.Name, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Failed to delete GPU %q: %v", created1.Name, err) + } + + _, err = client.DeviceV1alpha1().GPUs().Get(ctx, created1.Name, metav1.GetOptions{}) + if err == nil { + t.Errorf("expected error when getting deleted GPU %q, but got nil", created1.Name) + } + + err = client.DeviceV1alpha1().GPUs().Delete(ctx, created1.Name, metav1.DeleteOptions{}) + if err == nil { + t.Error("expected error when deleting already-deleted GPU, but got nil") + } + + list, err := client.DeviceV1alpha1().GPUs().List(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatalf("Failed to list GPUs after delete: %v", err) + } + + expectedCount := 2 + if len(list.Items) != expectedCount { + t.Errorf("expected %d GPUs remaining, got %d", expectedCount, len(list.Items)) + } + + for _, item := range list.Items { + if item.Name == created1.Name { + t.Errorf("deleted GPU %q still present in list", created1.Name) + } + } + + lastObjectRV, _ := strconv.ParseUint(created3.ResourceVersion, 10, 64) + listRV, _ := strconv.ParseUint(list.ResourceVersion, 10, 64) + if listRV < lastObjectRV { + t.Errorf("ResourceVersion (%d) is behind the last object's RV (%d)", listRV, lastObjectRV) + } + }) +} diff --git a/test/integration/device/v1alpha1/suite_test.go b/test/integration/device/v1alpha1/suite_test.go new file mode 100644 index 000000000..966688e99 --- /dev/null +++ b/test/integration/device/v1alpha1/suite_test.go @@ -0,0 +1,103 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1_test + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/nvidia/nvsentinel/cmd/device-apiserver/app" + "github.com/nvidia/nvsentinel/cmd/device-apiserver/app/options" + "github.com/nvidia/nvsentinel/pkg/client-go/client/versioned" + "github.com/nvidia/nvsentinel/pkg/grpc/client" + "github.com/nvidia/nvsentinel/pkg/util/testutils" +) + +var ( + clientset versioned.Interface + serverCtx context.Context + serverCancel context.CancelFunc +) + +func TestMain(m *testing.M) { + serverCtx, serverCancel = context.WithCancel(context.Background()) + defer serverCancel() + + tmpDir, _ := os.MkdirTemp("", "nvsentinel-test-*") + defer os.RemoveAll(tmpDir) + + socketPath, socketDir, err := testutils.CreateUnixAddr() + if err != nil { + fmt.Printf("Failed to create socket path: %v\n", err) + os.Exit(1) + } + defer os.RemoveAll(socketDir) + + kineSocketPath, kineDir, err := testutils.CreateUnixAddr() + if err != nil { + fmt.Printf("Failed to create kine socket path: %v\n", err) + os.Exit(1) + } + defer os.RemoveAll(kineDir) + + healthAddr, err := testutils.GetFreeTCPAddress() + if err != nil { + fmt.Printf("Failed to get free TCP port: %v\n", err) + os.Exit(1) + } + + opts := options.NewServerRunOptions() + opts.NodeName = "test-node" + opts.GRPC.BindAddress = "unix://" + socketPath + opts.HealthAddress = healthAddr + opts.Storage.DatabaseDir = tmpDir + opts.Storage.DatabasePath = tmpDir + "/state.db" + opts.Storage.KineSocketPath = kineSocketPath + opts.Storage.KineConfig.Endpoint = fmt.Sprintf("sqlite://%s/db.sqlite", tmpDir) + opts.Storage.KineConfig.Listener = "unix://" + kineSocketPath + + completed, err := opts.Complete(serverCtx) + if err != nil { + fmt.Printf("Failed to complete options: %v\n", err) + os.Exit(1) + } + + go func() { + if err := app.Run(serverCtx, completed); err != nil && err != context.Canceled { + fmt.Printf("Server exited with error: %v\n", err) + } + }() + + err = testutils.PollHealthStatus(healthAddr, "", 5*time.Second, testutils.IsServing) + if err != nil { + fmt.Printf("Server timed out waiting for health check: %v\n", err) + os.Exit(1) + } + + config := &client.Config{Target: "unix://" + socketPath} + clientset, err = versioned.NewForConfig(config) + if err != nil { + fmt.Printf("Failed to create clientset: %v\n", err) + os.Exit(1) + } + + code := m.Run() + + serverCancel() + os.Exit(code) +}