diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..d62b504ed --- /dev/null +++ b/.gitattributes @@ -0,0 +1,35 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ============================================================================== +# GIT ATTRIBUTES +# ============================================================================== +# Use 'linguist-generated=true' to hide generated code from GitHub PR diffs. +# ============================================================================== + +# Hide Kubernetes generated helpers (DeepCopy, Defaults, Conversions, OpenAPI, etc) +zz_generated.*.go linguist-generated=true + +# Hide generated Protobuf Go bindings +*.pb.go linguist-generated=true + +# Hide generated client library +client-go/client/** linguist-generated=true +client-go/listers/** linguist-generated=true +client-go/informers/** linguist-generated=true + +# Hide copied, unmodified upstream code +code-generator/cmd/client-gen/types/** linguist-generated=true +code-generator/cmd/client-gen/generators/scheme/** linguist-generated=true +code-generator/cmd/client-gen/generators/util/** linguist-generated=true diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index b874ea5c7..d26383ce9 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -1,260 +1,115 @@ -# GitHub Copilot Instructions for NVSentinel +# Copilot Instructions for NVSentinel -## Project Overview - -NVSentinel is a GPU Node Resilience System for Kubernetes that automatically detects, classifies, and remediates hardware and software faults in GPU nodes. It's designed for high-performance computing environments running NVIDIA GPUs. +This file provides repository-level instructions for GitHub Copilot to improve +code reviews and suggestions. -**Status**: Experimental/Preview Release - APIs and configurations may change. +## Project Overview -## Architecture & Technologies +NVSentinel is the NVIDIA Device API — a Protocol Buffer and gRPC-based API for +GPU device management. The codebase consists primarily of `.proto` definitions +and their generated Go code. -### Core Technologies -- **Language**: Go 1.25+ (primary), Python 3.10+ (monitoring tools) -- **Container Platform**: Kubernetes 1.25+ -- **Deployment**: Helm 3.0+, Tilt (development) -- **Storage**: MongoDB (event store with change streams) -- **Communication**: gRPC with Protocol Buffers -- **GPU Monitoring**: NVIDIA DCGM (Data Center GPU Manager) +## Repository Structure -### Project Structure ``` -├── health-monitors/ # Pluggable health detection modules -│ ├── gpu-health-monitor/ # DCGM-based GPU monitoring (Python) -│ ├── csp-health-monitor/ # Cloud provider health checks (Go) -│ └── syslog-health-monitor/ # System log analysis (Go) -├── health-events-analyzer/ # Event classification and routing -├── fault-quarantine/ # Node isolation (cordon) -├── node-drainer/ # Workload eviction -├── fault-remediation/ # Break-fix automation -├── labeler/ # Node labeling (DCGM version, driver status, Kata detection) -├── janitor/ # State cleanup and maintenance -├── platform-connectors/ # CSP integration (GCP, AWS, Azure) -├── commons/ # Shared utilities -├── data-models/ # Protocol Buffer definitions -├── store-client/ # MongoDB client library -└── distros/kubernetes/ # Helm charts +api/ +├── proto/ # Protocol Buffer source definitions (edit these) +│ └── device/v1alpha1/ +│ └── gpu.proto +├── gen/go/ # Generated Go code (DO NOT edit manually) +│ └── device/v1alpha1/ +│ ├── gpu.pb.go +│ └── gpu_grpc.pb.go +├── go.mod +└── Makefile ``` -## Coding Standards - -### Go Code Guidelines - -#### Module Organization -- Each service is a separate Go module with its own `go.mod` -- Use semantic import versioning -- Keep dependencies minimal and up-to-date -- Use `commons/` for shared utilities across modules - -#### Code Style -- Follow standard Go conventions (gofmt, golint) -- Use structured logging via `log/slog` -- Error handling: wrap errors with context using `fmt.Errorf("context: %w", err)` -- Within `retry.RetryOnConflict` blocks, return errors **without wrapping** to preserve retry behavior -- Use meaningful variable names (`synced` over `ok` for cache sync checks) - -#### Kubernetes Integration -- Use `client-go` for Kubernetes API interactions -- Prefer informers over direct API calls for watching resources -- Use `envtest` for testing Kubernetes controllers (not fake clients) -- Implement proper shutdown handling with context cancellation - -#### Testing Requirements -- Use `testify/assert` and `testify/require` for assertions -- Write table-driven tests when testing multiple scenarios -- Use `envtest` for integration tests with real Kubernetes API -- Test coverage: aim for >80% on critical paths -- Name tests descriptively: `TestFunctionName_Scenario_ExpectedBehavior` - -### Python Code Guidelines -- Use Poetry for dependency management -- Follow PEP 8 style guide -- Use Black for formatting -- Type hints required for all functions -- Use dataclasses for structured data - -### Protobuf Guidelines -- Define messages in `data-models/protobufs/` -- Use semantic versioning for breaking changes -- Include comprehensive comments for all fields -- Generate code with: `make protos-generate` - -## Development Workflows - -### Building & Testing -```bash -# Lint and test all modules -make lint-test-all +## Code Review Guidelines -# Lint specific module -cd labeler && make lint +### Protocol Buffers -# Test specific module -cd health-events-analyzer && make test +When reviewing `.proto` files: -# Build container images (uses ko) -make images +- **Field naming**: Use `snake_case` for field names +- **Message/Service naming**: Use `CamelCase` +- **Documentation**: Every message, field, and RPC must have documentation + comments explaining purpose and usage +- **Field numbers**: Never reuse or change field numbers in existing messages +- **Backwards compatibility**: New fields should be optional; avoid breaking + changes to existing APIs +- **Style**: Follow [Google's Protocol Buffer Style Guide](https://protobuf.dev/programming-guides/style/) -# View all make targets -make help -``` +### Generated Code -### Version Management -- All tool versions centralized in `.versions.yaml` -- Use `yq` to read versions in scripts -- Update versions in one place, propagates everywhere -- Check versions: `make show-versions` +- Files in `api/gen/` are **auto-generated** — do not suggest edits to these +- If generated code appears outdated, suggest running `make protos-generate` + +### Go Code + +- Use `gofmt` formatting (enforced by golangci-lint) +- Documentation comments should wrap at 80 characters +- Follow standard Go idioms and error handling patterns +- No manual edits to `*.pb.go` or `*_grpc.pb.go` files + +## Commit Standards + +### Conventional Commits + +All commits must follow conventional commit format: -### Local Development with Tilt -```bash -cd tilt -tilt up # Start local development environment ``` +feat: add new GPU condition type +fix: correct timestamp handling in conditions +docs: update API documentation +chore: update protoc-gen-go version +``` + +### DCO Sign-off + +All commits **must** include a DCO sign-off line: -## Kata Containers Detection - -The labeler implements Kata Containers detection: - -### Detection Architecture -- **Input labels** (on nodes): `katacontainers.io/kata-runtime` (default) + optional custom label -- **Output label** (set by labeler): `nvsentinel.dgxc.nvidia.com/kata.enabled: "true"|"false"` -- **Truthy values**: `"true"`, `"enabled"`, `"1"`, `"yes"` (case-insensitive) -- **Lifecycle separation**: Pod events → DCGM/driver labels, Node events → kata labels - -### DaemonSet Variants -- Separate DaemonSets for kata vs regular nodes -- Selection via `nodeAffinity` based on kata.enabled label -- Different volume mounts: - - Regular: `/var/log` (file-based logs) - - Kata: `/run/log/journal` and `/var/log/journal` (systemd journal) - -## Important Patterns - -### Error Handling in Retry Loops -```go -// ✅ CORRECT - Return error as-is for retry logic -err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - _, err := client.Update(ctx, obj, metav1.UpdateOptions{}) - return err // Don't wrap! -}) - -// ❌ WRONG - Wrapping breaks retry detection -err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - _, err := client.Update(ctx, obj, metav1.UpdateOptions{}) - return fmt.Errorf("failed: %w", err) // Breaks retry! -}) ``` +Signed-off-by: Name +``` + +Use `git commit -s` to add this automatically. + +## License Headers + +All source files must include the Apache 2.0 license header: -### Informer Usage -```go -// Use separate informers for different resource types -podInformer := factory.Core().V1().Pods().Informer() -nodeInformer := factory.Core().V1().Nodes().Informer() - -// Extract event handler setup into helper methods -func (l *Labeler) registerPodEventHandlers() error { - _, err := l.podInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ - // handlers... - }) - return err -} ``` +Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -### Testing with envtest -```go -// Use real Kubernetes API server for tests -testEnv := &envtest.Environment{ - CRDDirectoryPaths: []string{}, -} -cfg, err := testEnv.Start() -// Create clients from cfg -// Run tests -testEnv.Stop() +Licensed under the Apache License, Version 2.0 (the "License"); +... ``` -## Documentation Standards - -### Code Comments -- Package-level godoc for all packages -- Function comments for exported functions -- Inline comments for complex logic only -- TODO comments should reference issues - -### Helm Chart Documentation -- Document all values in `values.yaml` with inline comments -- Include examples for non-obvious configurations -- Note truthy value requirements where applicable -- Explain DaemonSet variant selection logic - -## Security & Compliance - -### Licensing -- All files must include Apache 2.0 license header -- Use `make license-headers-add` to add headers -- No GPL or viral licenses in dependencies - -### Security -- Report security issues via SECURITY.md process -- Never commit secrets or credentials -- Use Kubernetes secrets for sensitive data -- Scan dependencies: automated via CI - -## CI/CD Pipeline - -### GitHub Actions Workflows -- **PR Checks**: lint, test, build for all modules -- **Release**: automated image builds and Helm chart publishing -- **Vulnerability Scanning**: daily Trivy scans uploaded to Security tab -- **Provenance**: SLSA attestation for all artifacts - -### Container Images -- Built with `ko` (Go) and `docker` (Python) -- Multi-architecture support (amd64, arm64) -- Published to `ghcr.io/nvidia/nvsentinel/*` -- Signed with Sigstore cosign - -## Common Tasks - -### Adding a New Health Monitor -1. Create directory in `health-monitors/` -2. Implement gRPC service from `data-models/protobufs/` -3. Add Makefile with standard targets -4. Create Helm chart in `distros/kubernetes/nvsentinel/charts/` -5. Register in platform-connectors for CSP integration -6. Add to CI pipeline workflows - -### Updating Dependencies -```bash -# Update Go dependencies -make gomod-tidy +## What to Flag in Reviews -# Update Python dependencies -cd health-monitors/gpu-health-monitor -poetry update -``` +1. **Breaking API changes** without migration path +2. **Missing documentation** on proto messages/fields/RPCs +3. **Manual edits** to generated files +4. **Missing license headers** +5. **Unsigned commits** (missing DCO) +6. **Non-conventional commit messages** -### Adding New Node Labels -1. Update labeler logic in `pkg/labeler/labeler.go` -2. Add tests in `labeler_test.go` -3. Document in Helm chart values -4. Update KATA_TESTING.md if kata-related +## What NOT to Flag -## Debugging Tips +1. Style issues in generated `*.pb.go` files +2. Import ordering in generated code +3. Line length in generated code -### Local Development -- Use Tilt for rapid iteration -- Check logs: `kubectl logs -n nvsentinel ` -- Port-forward for direct access: `kubectl port-forward -n nvsentinel svc/janitor 8080:8080` -- MongoDB Compass for event store inspection +## Build Commands -### Common Issues -- **Informer not syncing**: Check RBAC permissions -- **Labels not updating**: Verify labeler is running and has node permissions -- **DaemonSet not scheduling**: Check node selectors and labels -- **Kata detection failing**: Verify input labels on nodes +```bash +make protos-generate # Regenerate Go code from .proto files +make build # Build the Go module +make lint # Run go vet +make test # Run tests +``` -## References +## Tool Versions -- [DEVELOPMENT.md](../DEVELOPMENT.md) - Detailed development guide -- [CONTRIBUTING.md](../CONTRIBUTING.md) - Contribution guidelines -- [README.md](../README.md) - Project overview -- [docs/](../docs/) - Architecture documentation +Tool versions are centralized in `.versions.yaml`. When suggesting dependency +updates, reference this file. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..49e8c1f74 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,230 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: CI + +on: + push: + branches: + - device-api + pull_request: + branches: + - device-api + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +permissions: + contents: read # Required for checking out code + actions: write # Required for uploading test artifacts + pull-requests: write # Required for coverage report comments + +env: + # Go cache settings (specific to this workflow) + GOPATH: /home/runner/go + GOCACHE: /home/runner/.cache/go-build + +jobs: + go: + name: Go - ${{ matrix.step_name }} + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - component: api + step_name: "API Module" + make_command: "make -C api lint test build" + - component: client-go + step_name: "Client-Go Module" + make_command: "make -C client-go lint test build" + - component: code-generator + step_name: "Code Generator" + make_command: "make -C code-generator build" + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup build environment + uses: ./.github/actions/setup-ci-env + + - name: ${{ matrix.step_name }} + run: ${{ matrix.make_command }} + + health-monitors-lint-test: + if: github.repository == 'nvidia/nvsentinel' + runs-on: linux-amd64-cpu16 + timeout-minutes: 30 + strategy: + matrix: + include: + - component: syslog-health-monitor + - component: csp-health-monitor + - component: kubernetes-object-monitor + - component: gpu-health-monitor + install_dcgm: 'true' + python_required: 'true' + replace_imports: 'false' + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup build environment + uses: ./.github/actions/setup-ci-env + + - name: Run lint and test + run: make -C health-monitors/${{ matrix.component }} lint-test + + - name: Upload artifacts + uses: ./.github/actions/upload-test-artifacts + with: + component-name: ${{ matrix.component }} + file-paths: | + health-monitors/${{ matrix.component }}/coverage.xml + health-monitors/${{ matrix.component }}/coverage.txt + health-monitors/${{ matrix.component }}/report.xml + + preflight-checks-lint-test: + if: github.repository == 'nvidia/nvsentinel' + runs-on: linux-amd64-cpu16 + timeout-minutes: 30 + strategy: + matrix: + component: + - ping + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup build environment + uses: ./.github/actions/setup-ci-env + + - name: Run lint and test + run: make -C preflight-checks/${{ matrix.component }} lint-test + + - name: Upload artifacts + uses: ./.github/actions/upload-test-artifacts + with: + component-name: preflight-${{ matrix.component }} + file-paths: | + preflight-checks/${{ matrix.component }}/coverage.xml + preflight-checks/${{ matrix.component }}/coverage.txt + preflight-checks/${{ matrix.component }}/report.xml + + modules-lint-test: + if: github.repository == 'nvidia/nvsentinel' + runs-on: linux-amd64-cpu16 + timeout-minutes: 30 + strategy: + matrix: + component: + - platform-connectors + - event-exporter + - store-client + - commons + - data-models + - health-events-analyzer + - fault-quarantine + - labeler + - metadata-collector + - node-drainer + - fault-remediation + - janitor + - preflight + - tests + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup build environment + uses: ./.github/actions/setup-ci-env + + - name: Run lint and test + run: make -C ${{ matrix.component }} lint-test + + - name: Upload artifacts + uses: ./.github/actions/upload-test-artifacts + with: + component-name: ${{ matrix.component }} + file-paths: | + ${{ matrix.component }}/coverage.xml + ${{ matrix.component }}/coverage.txt + ${{ matrix.component }}/report.xml + + tilt-modules-lint-test: + if: github.repository == 'nvidia/nvsentinel' + runs-on: linux-amd64-cpu16 + timeout-minutes: 30 + strategy: + matrix: + component: + - tilt/simple-health-client + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup build environment + uses: ./.github/actions/setup-ci-env + + - name: Run lint and test + run: make -C ${{ matrix.component }} lint-test + + - name: Upload artifacts + uses: ./.github/actions/upload-test-artifacts + with: + component-name: simple-health-client + file-paths: | + ${{ matrix.component }}/coverage.xml + ${{ matrix.component }}/coverage.txt + ${{ matrix.component }}/report.xml + + consolidated-coverage-report: + if: github.repository == 'nvidia/nvsentinel' && (github.event_name == 'pull_request' || startsWith(github.ref, 'refs/heads/pull-request/')) + runs-on: linux-amd64-cpu16 + timeout-minutes: 15 + needs: [health-monitors-lint-test, preflight-checks-lint-test, modules-lint-test, tilt-modules-lint-test] + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup Go + uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 + with: + go-version-file: api/go.mod + cache-dependency-path: "**/go.sum" + + - name: Install tools + run: | + echo "Installing yq..." + sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 + sudo chmod +x /usr/local/bin/yq + + echo "Installing protoc..." + PROTOC_VERSION=$(yq eval '.protobuf.protobuf' .versions.yaml | sed 's/v//') + curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip + sudo unzip -o protoc-${PROTOC_VERSION}-linux-x86_64.zip -d /usr/local + rm protoc-${PROTOC_VERSION}-linux-x86_64.zip + + echo "Installing golangci-lint..." + LINT_VERSION=$(yq eval '.go_tools.golangci_lint' .versions.yaml) + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin $LINT_VERSION + + - name: Generate + run: make verify-codegen + + - name: Lint + run: make lint + + - name: Build + run: make build + + - name: Test + run: make test diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml deleted file mode 100644 index 9e469821a..000000000 --- a/.github/workflows/e2e-test.yml +++ /dev/null @@ -1,238 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: E2E Tests - -# This workflow runs end-to-end tests on both AMD64 and ARM64 architectures in parallel -# with both MongoDB and PostgreSQL datastores to ensure compatibility across different -# hardware platforms and database backends. -# -# Configuration: -# - Set RUNNER_ARCH_LARGE_AMD64 variable to override default AMD64 runner -# - Set RUNNER_ARCH_LARGE_ARM64 variable to override default ARM64 runner -# - Each architecture + datastore combination gets its own isolated cluster and test artifacts - -on: - push: - branches: - - main - - "pull-request/[0-9]+" - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -permissions: - contents: read # Required for checking out code - actions: read # Required for artifact operations - -jobs: - e2e-test: - if: github.repository == 'nvidia/nvsentinel' - # Run E2E tests on both AMD64 and ARM64 architectures with both MongoDB and PostgreSQL - strategy: - fail-fast: false # Allow all combinations to complete even if some fail - matrix: - include: - - arch: amd64 - runner: ${{ vars.RUNNER_ARCH_LARGE_AMD64 || 'linux-amd64-cpu32' }} - arch_name: "AMD64" - datastore: "mongodb" - datastore_name: "MongoDB" - - arch: arm64 - runner: ${{ vars.RUNNER_ARCH_LARGE_ARM64 || 'linux-arm64-cpu32' }} - arch_name: "ARM64" - datastore: "mongodb" - datastore_name: "MongoDB" - - arch: amd64 - runner: ${{ vars.RUNNER_ARCH_LARGE_AMD64 || 'linux-amd64-cpu32' }} - arch_name: "AMD64" - datastore: "postgresql" - datastore_name: "PostgreSQL" - - arch: arm64 - runner: ${{ vars.RUNNER_ARCH_LARGE_ARM64 || 'linux-arm64-cpu32' }} - arch_name: "ARM64" - datastore: "postgresql" - datastore_name: "PostgreSQL" - - name: "E2E Tests (${{ matrix.arch_name }} + ${{ matrix.datastore_name }})" - runs-on: ${{ matrix.runner }} - timeout-minutes: 120 - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Workaround for freeing up more disk space - run: | - sudo rm -rf /usr/local/lib/android - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf /opt/hostedtoolcache/CodeQL - sudo docker image prune --all --force - # Additional Docker cleanup as recommended by Kind - docker system prune -f - - - name: Setup build environment - uses: ./.github/actions/setup-ci-env - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - - - name: Prep system for multi-node Kind cluster - run: | - # System configuration for Kind multi-node setup - sudo systemctl stop apparmor || echo "unable to stop apparmor" - sudo systemctl disable apparmor || echo "unable to disable apparmor" - sudo modprobe br_netfilter || echo "unable to run modprobe" - - # Network configuration - sudo sysctl -w net.ipv6.conf.all.forwarding=1 - sudo sysctl -w net.ipv4.ip_forward=1 - sudo sysctl -w net.bridge.bridge-nf-call-ip6tables=1 - sudo sysctl -w net.bridge.bridge-nf-call-iptables=1 - - # File system limits for Kind - sudo sysctl -w fs.inotify.max_user_watches=524288 - sudo sysctl -w fs.inotify.max_user_instances=1024 - - # IPTables cleanup and configuration - sudo iptables -F && sudo iptables -X && sudo iptables -t nat -F && sudo iptables -t nat -X && sudo iptables -t mangle -F && sudo iptables -t mangle -X && sudo iptables -P INPUT ACCEPT && sudo iptables -P FORWARD ACCEPT -w 5 && sudo iptables -P OUTPUT ACCEPT -w 5 - sudo systemctl restart docker - - - name: Install E2E testing tools - uses: ./.github/actions/install-e2e-tools - - - name: Configure Helm repositories - run: | - helm repo add prometheus-community https://prometheus-community.github.io/helm-charts - helm repo add bitnami https://charts.bitnami.com/bitnami - helm repo add jetstack https://charts.jetstack.io - helm repo update - - - name: Configure ctlptl registry authentication - run: | - ./scripts/configure-ctlptl-registry.sh - - - name: Compute ref name with short SHA - id: ref-name - run: | - SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7) - SAFE_REF="${{ github.ref_name }}-${SHORT_SHA}" - # Sanitize ref name: replace slashes with hyphens for Docker tag compatibility - SAFE_REF=$(echo "$SAFE_REF" | sed 's/\//-/g') - echo "value=$SAFE_REF" >> $GITHUB_OUTPUT - - - name: Create cluster for E2E tests - env: - CI_COMMIT_REF_NAME: ${{ steps.ref-name.outputs.value }} - CTLPTL_YAML: .ctlptl.yaml - # Make cluster names unique per architecture and datastore to avoid conflicts in parallel runs - CLUSTER_NAME_SUFFIX: "-${{ matrix.arch }}-${{ matrix.datastore }}" - run: | - make cluster-create - - - name: Override MongoDB image for ARM64 - if: matrix.arch == 'arm64' && matrix.datastore == 'mongodb' - run: | - sed -i 's/repository: "bitnamilegacy\/mongodb"/repository: "dlavrenuek\/bitnami-mongodb-arm"/' distros/kubernetes/nvsentinel/values-tilt.yaml - sed -i 's/tag: "8.0.3-debian-12-r1"/tag: "8.0.4"/' distros/kubernetes/nvsentinel/values-tilt.yaml - - - name: Run E2E tests - env: - CI_COMMIT_REF_NAME: ${{ steps.ref-name.outputs.value }} - CTLPTL_YAML: .ctlptl.yaml - # Use same cluster name suffix for consistency - CLUSTER_NAME_SUFFIX: "-${{ matrix.arch }}-${{ matrix.datastore }}" - # Set USE_POSTGRESQL for PostgreSQL tests (our integrated Tiltfile approach) - USE_POSTGRESQL: ${{ matrix.datastore == 'postgresql' && '1' || '0' }} - # Set architecture-specific test tags - TEST_TAGS: ${{ matrix.arch }}_group${{ matrix.datastore == 'mongodb' && ',mongodb' || '' }} - run: | - make e2e-test-ci - - - name: Upload test results - uses: ./.github/actions/upload-test-artifacts - with: - component-name: e2e-test-${{ matrix.arch }}-${{ matrix.datastore }} - file-paths: | - tests/results/ - tests/*.log - retention-days: 14 - - - name: Export Kind logs - if: always() - run: | - mkdir -p /tmp/kind-logs - CLUSTER_NAME=$(kind get clusters | head -n1) - if [ -n "$CLUSTER_NAME" ]; then - kind export logs /tmp/kind-logs --name "$CLUSTER_NAME" || true - fi - - - name: Pull Kind container logs via crictl - if: always() - run: | - chmod +x ./scripts/pull_kind_logs.sh - ./scripts/pull_kind_logs.sh /tmp/kind-container-logs || true - - - name: Upload Kind container logs - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: e2e-kind-container-logs-${{ matrix.arch }}-${{ matrix.datastore }}-${{ github.run_id }} - path: /tmp/kind-container-logs/ - retention-days: 7 - - - name: Collect debug artifacts - if: failure() - run: | - mkdir -p /tmp/debug-artifacts - kubectl get all --all-namespaces > /tmp/debug-artifacts/all-resources.yaml || true - kubectl get events --all-namespaces --sort-by='.lastTimestamp' > /tmp/debug-artifacts/all-events.yaml || true - kubectl get pods --all-namespaces -o yaml > /tmp/debug-artifacts/all-pods.yaml || true - kubectl get jobs --all-namespaces -o yaml > /tmp/debug-artifacts/all-jobs.yaml || true - kubectl logs --all-namespaces --all-containers=true --tail=500 > /tmp/debug-artifacts/all-logs.txt || true - docker images > /tmp/debug-artifacts/docker-images.txt || true - df -h > /tmp/debug-artifacts/disk-usage.txt || true - # Comprehensive cluster state dump for debugging - kubectl cluster-info dump --all-namespaces --output-directory=/tmp/debug-artifacts/cluster-info-dump || true - # Get node details including cordon state - kubectl get nodes -o wide > /tmp/debug-artifacts/nodes-wide.txt || true - kubectl get nodes -o yaml > /tmp/debug-artifacts/nodes-full.yaml || true - # Get node descriptions for annotations and conditions - kubectl describe nodes > /tmp/debug-artifacts/nodes-describe.txt || true - - - name: Upload Kind logs - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: e2e-kind-logs-${{ matrix.arch }}-${{ matrix.datastore }}-${{ github.run_id }} - path: /tmp/kind-logs/ - retention-days: 7 - - - name: Upload debug artifacts - if: failure() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: e2e-debug-artifacts-${{ matrix.arch }}-${{ matrix.datastore }}-${{ github.run_id }} - path: /tmp/debug-artifacts/ - retention-days: 7 - - - name: Cleanup Docker resources - if: always() - run: | - docker rm -f $(docker ps -a -q) || true - docker rmi -f $(docker images -q -a) || true - docker volume prune -f || true - docker network prune -f || true - docker builder prune -f || true diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml deleted file mode 100644 index d3bdb1e57..000000000 --- a/.github/workflows/lint-test.yml +++ /dev/null @@ -1,489 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Lint and Test - -on: - push: - branches: - - main - - "pull-request/[0-9]+" - tags: - - 'v*' - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -permissions: - contents: read # Required for checking out code - actions: write # Required for uploading test artifacts - pull-requests: write # Required for coverage report comments - -env: - # Go cache settings (specific to this workflow) - GOPATH: /home/runner/go - GOCACHE: /home/runner/.cache/go-build - -jobs: - simple-lint: - if: github.repository == 'nvidia/nvsentinel' - runs-on: linux-amd64-cpu16 - timeout-minutes: 30 - strategy: - matrix: - include: - - component: protos - make_command: 'make protos-lint' - step_name: 'Run protos lint' - - component: license-headers - make_command: 'make license-headers-lint' - step_name: 'Run license headers check' - - component: gomod - make_command: 'make gomod-lint' - step_name: 'Run gomod lint' - - component: log-collector - make_command: 'make -C log-collector lint-log-collector' - step_name: 'Run lint' - replace_imports: 'false' - - component: gpu-reset - make_command: 'make -C gpu-reset lint' - step_name: 'Run lint' - replace_imports: 'false' - - component: file-server-cleanup - make_command: 'make -C log-collector lint-file-server-cleanup' - step_name: 'Run lint' - replace_imports: 'false' - - component: kubernetes-distro - make_command: 'make kubernetes-distro-lint' - step_name: 'Run lint' - - component: helm-charts - make_command: 'make helm-lint' - step_name: 'Validate Helm charts' - - component: postgres-schema - make_command: 'make validate-postgres-schema' - step_name: 'Validate PostgreSQL schema consistency' - - component: scripts - make_command: 'make -C scripts lint' - step_name: 'Run shellcheck on scripts' - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup build environment - uses: ./.github/actions/setup-ci-env - - - name: ${{ matrix.step_name }} - run: ${{ matrix.make_command }} - - - name: Load Helm version from .versions.yaml - if: matrix.component == 'helm-charts' - id: helm-version - run: | - HELM_VERSION=$(yq eval '.testing_tools.helm' .versions.yaml) - echo "helm_version=${HELM_VERSION}" >> $GITHUB_OUTPUT - - - name: Setup Helm - if: matrix.component == 'helm-charts' - uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1 - with: - version: ${{ steps.helm-version.outputs.helm_version }} - - - name: Validate Helm Charts - if: matrix.component == 'helm-charts' - run: make helm-lint - - health-monitors-lint-test: - if: github.repository == 'nvidia/nvsentinel' - runs-on: linux-amd64-cpu16 - timeout-minutes: 30 - strategy: - matrix: - include: - - component: syslog-health-monitor - - component: csp-health-monitor - - component: kubernetes-object-monitor - - component: gpu-health-monitor - install_dcgm: 'true' - python_required: 'true' - replace_imports: 'false' - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup build environment - uses: ./.github/actions/setup-ci-env - - - name: Run lint and test - run: make -C health-monitors/${{ matrix.component }} lint-test - - - name: Upload artifacts - uses: ./.github/actions/upload-test-artifacts - with: - component-name: ${{ matrix.component }} - file-paths: | - health-monitors/${{ matrix.component }}/coverage.xml - health-monitors/${{ matrix.component }}/coverage.txt - health-monitors/${{ matrix.component }}/report.xml - - preflight-checks-lint-test: - if: github.repository == 'nvidia/nvsentinel' - runs-on: linux-amd64-cpu16 - timeout-minutes: 30 - strategy: - matrix: - component: - - ping - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup build environment - uses: ./.github/actions/setup-ci-env - - - name: Run lint and test - run: make -C preflight-checks/${{ matrix.component }} lint-test - - - name: Upload artifacts - uses: ./.github/actions/upload-test-artifacts - with: - component-name: preflight-${{ matrix.component }} - file-paths: | - preflight-checks/${{ matrix.component }}/coverage.xml - preflight-checks/${{ matrix.component }}/coverage.txt - preflight-checks/${{ matrix.component }}/report.xml - - modules-lint-test: - if: github.repository == 'nvidia/nvsentinel' - runs-on: linux-amd64-cpu16 - timeout-minutes: 30 - strategy: - matrix: - component: - - platform-connectors - - event-exporter - - store-client - - commons - - data-models - - health-events-analyzer - - fault-quarantine - - labeler - - metadata-collector - - node-drainer - - fault-remediation - - janitor - - preflight - - tests - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup build environment - uses: ./.github/actions/setup-ci-env - - - name: Run lint and test - run: make -C ${{ matrix.component }} lint-test - - - name: Upload artifacts - uses: ./.github/actions/upload-test-artifacts - with: - component-name: ${{ matrix.component }} - file-paths: | - ${{ matrix.component }}/coverage.xml - ${{ matrix.component }}/coverage.txt - ${{ matrix.component }}/report.xml - - tilt-modules-lint-test: - if: github.repository == 'nvidia/nvsentinel' - runs-on: linux-amd64-cpu16 - timeout-minutes: 30 - strategy: - matrix: - component: - - tilt/simple-health-client - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup build environment - uses: ./.github/actions/setup-ci-env - - - name: Run lint and test - run: make -C ${{ matrix.component }} lint-test - - - name: Upload artifacts - uses: ./.github/actions/upload-test-artifacts - with: - component-name: simple-health-client - file-paths: | - ${{ matrix.component }}/coverage.xml - ${{ matrix.component }}/coverage.txt - ${{ matrix.component }}/report.xml - - consolidated-coverage-report: - if: github.repository == 'nvidia/nvsentinel' && (github.event_name == 'pull_request' || startsWith(github.ref, 'refs/heads/pull-request/')) - runs-on: linux-amd64-cpu16 - timeout-minutes: 15 - needs: [health-monitors-lint-test, preflight-checks-lint-test, modules-lint-test, tilt-modules-lint-test] - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup Go - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 - with: - go-version: 'stable' - - - name: Download all coverage artifacts - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 - with: - pattern: "*-results" - path: coverage-artifacts - merge-multiple: false - - - name: Consolidate coverage files - run: | - set -e - echo "Consolidating coverage files from all components..." - mkdir -p consolidated-coverage - - # Initialize consolidated coverage with mode line - echo "mode: set" > consolidated-coverage/coverage.txt - - # Find all coverage.txt files and merge them properly - find coverage-artifacts -name "coverage.txt" -type f | while read -r file; do - echo "Processing: $file" - - # Validate file exists and is not empty - if [[ ! -f "$file" || ! -s "$file" ]]; then - echo "Warning: Skipping empty or missing file: $file" - continue - fi - - # Validate coverage file format - if ! head -n 1 "$file" | grep -q "^mode:"; then - echo "Warning: Skipping file with invalid format (no mode line): $file" - continue - fi - - # Extract coverage data (skip mode line) and validate each line - tail -n +2 "$file" | while IFS= read -r line; do - # Skip empty lines - [[ -z "$line" ]] && continue - - # Validate coverage line format: file.go:start.col,end.col numStmts count - if [[ "$line" =~ ^[^:]+:[0-9]+\.[0-9]+,[0-9]+\.[0-9]+[[:space:]]+[0-9]+[[:space:]]+[0-9]+$ ]]; then - echo "$line" >> consolidated-coverage/coverage.txt - else - echo "Warning: Skipping malformed coverage line: $line" - fi - done - done - - echo "✅ Coverage consolidation completed successfully" - - - name: Upload consolidated coverage - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: consolidated-code-coverage - path: consolidated-coverage/coverage.txt - retention-days: 30 - - - name: Extract PR number from branch name - id: pr-number - run: | - if [[ "${{ github.ref }}" =~ pull-request/([0-9]+) ]]; then - echo "pr_number=${BASH_REMATCH[1]}" >> $GITHUB_OUTPUT - else - echo "pr_number=" >> $GITHUB_OUTPUT - fi - - - name: Install go-coverage-report CLI tool - run: go install github.com/fgrosse/go-coverage-report/cmd/go-coverage-report@v1.2.0 - - - name: Get changed files - uses: tj-actions/changed-files@aa08304bd477b800d468db44fe10f6c61f7f7b11 - id: changed-files - with: - write_output_files: true - json: true - files: "**.go" - files_ignore: "vendor/**" - output_dir: .github/outputs - - - name: Generate Coverage Report with Fixed PR Number - if: steps.changed-files.outputs.any_changed == 'true' - run: | - set -e # Exit on error - - # Use the locally consolidated coverage file (no download needed!) - echo "Using locally consolidated coverage..." - mkdir -p .github/outputs - - if [[ ! -f consolidated-coverage/coverage.txt || ! -s consolidated-coverage/coverage.txt ]]; then - echo "❌ Consolidated coverage file not found or empty" - exit 1 - fi - - # Copy (don't move) so the artifact upload still has the original - cp consolidated-coverage/coverage.txt .github/outputs/new-coverage.txt - echo "✅ Current coverage prepared from local file" - - # Download baseline coverage from main (failure here is acceptable) - echo "Downloading baseline coverage..." - LAST_SUCCESSFUL_RUN=$(gh run list --status=success --branch=main --workflow=lint-test.yml --event=push --json=databaseId --limit=1 -q '.[] | .databaseId') - - if [[ -n "$LAST_SUCCESSFUL_RUN" ]]; then - echo "Found baseline run: $LAST_SUCCESSFUL_RUN" - if gh run download "$LAST_SUCCESSFUL_RUN" --name=consolidated-code-coverage --dir=/tmp/baseline-coverage 2>/dev/null; then - if [[ -f /tmp/baseline-coverage/coverage.txt ]]; then - echo "✅ Baseline coverage found" - mv /tmp/baseline-coverage/coverage.txt .github/outputs/old-coverage.txt - else - echo "⚠️ Baseline coverage file not found in artifact" - touch .github/outputs/old-coverage.txt # Create empty file - fi - else - echo "⚠️ Failed to download baseline coverage (creating empty baseline)" - touch .github/outputs/old-coverage.txt # Create empty file - fi - else - echo "⚠️ No successful baseline run found (creating empty baseline)" - touch .github/outputs/old-coverage.txt # Create empty file - fi - - # Generate the report using fgrosse's CLI tool (same format!) - echo "Generating coverage report..." - if ! go-coverage-report -root=github.com/nvidia/nvsentinel \ - .github/outputs/old-coverage.txt \ - .github/outputs/new-coverage.txt \ - .github/outputs/all_modified_files.json > coverage-report.md 2> coverage-report.err; then - echo "❌ Failed to generate coverage report" - exit 1 - fi - - # Check if report is empty and why - if [[ ! -f coverage-report.md || ! -s coverage-report.md ]]; then - if grep -q "no changed files" coverage-report.err 2>/dev/null; then - echo "ℹ️ No Go files changed - skipping coverage report" - echo "## 📊 Coverage Report" > coverage-report.md - echo "No Go files were modified in this PR, so no coverage analysis is needed." >> coverage-report.md - else - echo "❌ Coverage report is empty or missing for unknown reason" - echo "Error output from go-coverage-report:" - cat coverage-report.err || echo "No error output" - exit 1 - fi - fi - - echo "✅ Coverage report generated successfully" - - # Check if coverage report indicates no change to avoid spam - if grep -q "will \*\*not change\*\* overall coverage" coverage-report.md; then - echo "ℹ️ Coverage report shows no change - skipping PR comment to reduce noise" - exit 0 - fi - - # Post comment using our correct PR number - if [[ -n "${{ steps.pr-number.outputs.pr_number }}" ]]; then - echo "Posting coverage comment to PR #${{ steps.pr-number.outputs.pr_number }}..." - - # Check for existing coverage comment - EXISTING_COMMENT=$(gh api "repos/${{ github.repository }}/issues/${{ steps.pr-number.outputs.pr_number }}/comments" \ - --jq '.[] | select(.user.login=="github-actions[bot]" and (.body | test("Coverage Report|Coverage Δ"))) | .id' \ - | head -1 2>/dev/null || echo "") - - if [[ -n "$EXISTING_COMMENT" ]]; then - echo "Updating existing comment $EXISTING_COMMENT..." - if ! gh api "repos/${{ github.repository }}/issues/${{ steps.pr-number.outputs.pr_number }}/comments/$EXISTING_COMMENT" \ - --method PATCH --input coverage-report.md; then - echo "⚠️ Failed to update existing comment (likely permissions), creating new comment instead..." - if ! gh pr comment "${{ steps.pr-number.outputs.pr_number }}" --body-file=coverage-report.md; then - echo "❌ Failed to create new coverage comment" - exit 1 - fi - fi - else - echo "Creating new comment..." - if ! gh pr comment "${{ steps.pr-number.outputs.pr_number }}" --body-file=coverage-report.md; then - echo "❌ Failed to create coverage comment" - exit 1 - fi - fi - - echo "✅ Coverage comment posted successfully" - else - echo "⚠️ No PR number found, skipping comment" - fi - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - consolidated-coverage-baseline: - if: github.repository == 'nvidia/nvsentinel' && github.ref == 'refs/heads/main' && github.event_name == 'push' - runs-on: linux-amd64-cpu16 - timeout-minutes: 15 - needs: [health-monitors-lint-test, preflight-checks-lint-test, modules-lint-test, tilt-modules-lint-test] - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Download all coverage artifacts - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 - with: - pattern: "*-results" - path: coverage-artifacts - merge-multiple: false - - - name: Consolidate coverage files - run: | - set -e - echo "Consolidating coverage files from all components for baseline..." - mkdir -p consolidated-coverage - - # Initialize consolidated coverage with mode line - echo "mode: set" > consolidated-coverage/coverage.txt - - # Find all coverage.txt files and merge them properly - find coverage-artifacts -name "coverage.txt" -type f | while read -r file; do - echo "Processing: $file" - - # Validate file exists and is not empty - if [[ ! -f "$file" || ! -s "$file" ]]; then - echo "Warning: Skipping empty or missing file: $file" - continue - fi - - # Validate coverage file format - if ! head -n 1 "$file" | grep -q "^mode:"; then - echo "Warning: Skipping file with invalid format (no mode line): $file" - continue - fi - - # Extract coverage data (skip mode line) and validate each line - tail -n +2 "$file" | while IFS= read -r line; do - # Skip empty lines - [[ -z "$line" ]] && continue - - # Validate coverage line format: file.go:start.col,end.col numStmts count - if [[ "$line" =~ ^[^:]+:[0-9]+\.[0-9]+,[0-9]+\.[0-9]+[[:space:]]+[0-9]+[[:space:]]+[0-9]+$ ]]; then - echo "$line" >> consolidated-coverage/coverage.txt - else - echo "Warning: Skipping malformed coverage line: $line" - fi - done - done - - echo "✅ Coverage consolidation completed successfully" - - - name: Upload consolidated coverage baseline - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: consolidated-code-coverage - path: consolidated-coverage/coverage.txt - retention-days: 90 # Keep baseline longer - diff --git a/.gitignore b/.gitignore index 6f355aad7..214ae8600 100644 --- a/.gitignore +++ b/.gitignore @@ -20,124 +20,21 @@ api/bin/* *.test # Output of the go coverage tool -*.out +**/*.out +code-quality-report.json # Go workspace file go.work go.work.sum -# Dependency directories -# vendor/ - -### Python ### -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions - -# Distribution / packaging -.Python -build/ -develop-eggs/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Python version management -# .python-version - -# Poetry local configuration file -poetry.toml - -# Pipenv -#Pipfile.lock - -# Poetry lock file -#poetry.lock - -# pdm -#pdm.lock -.pdm.toml -__pypackages__/ - -# ============================================================================ -# Testing & Coverage Reports -# ============================================================================ -# Test reports -report.xml -nosetests.xml -coverage.xml - -# Coverage data -.coverage -.coverage.* -*.cover -*.py,cover -cover/ -coverage.txt -htmlcov/ - -# Code quality reports -code-quality-report.json - -# Test frameworks -.tox/ -.nox/ -.cache -.hypothesis/ -.pytest_cache/ - -# ============================================================================ -# Documentation & Build Outputs -# ============================================================================ -# Sphinx documentation -docs/_build/ - -# mkdocs documentation -/site - -# PyBuilder -.pybuilder/ -target/ - -# Distribution directories -dist/ +# Server binary output +bin/ +device-api-server -# ============================================================================ +# ============================================================================== # IDE & Editor Configurations # ============================================================================ -### JetBrains IDEs (GoLand, PyCharm, IntelliJ) ### ### JetBrains IDEs (GoLand, PyCharm, IntelliJ) ### # User-specific stuff .idea/ @@ -249,198 +146,16 @@ tags ### Emacs ### *~ \#*\# -/.emacs.desktop -/.emacs.desktop.lock -*.elc -auto-save-list -tramp -.\#* - -# Org-mode -.org-id-locations -*_archive - -# flymake-mode -*_flymake.* - -# eshell files -/eshell/history -/eshell/lastdir - -# elpa packages -/elpa/ - -# reftex files -*.rel - -# AUCTeX auto folder -/auto/ - -# cask packages -.cask/ - -# Flycheck -flycheck_*.el - -# server auth directory -/server/ - -# projectiles files -.projectile - -# directory configuration -.dir-locals.el - -# network security -/network-security.data - -# ============================================================================ -# Notebooks & Interactive Development -# ============================================================================ -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# ============================================================================ -# Python Type Checkers & Linters -# ============================================================================ -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ -# Cython debug symbols -cython_debug/ - -# ruff -.ruff_cache/ - -# LSP config files -pyrightconfig.json - -# ============================================================================ -# Python Web Frameworks -# ============================================================================ -# Django -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask -instance/ -.webassets-cache - -# Scrapy -.scrapy - -# Celery -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# Translations -*.mo -*.pot - -# ============================================================================ -# Infrastructure as Code -# ============================================================================ - -### Terraform ### -# Local .terraform directories -**/.terraform/* - -# .tfstate files -*.tfstate -*.tfstate.* - -# Crash log files -crash.log -crash.*.log - -# Sensitive data files -*.tfvars -*.tfvars.json - -# Override files -override.tf -override.tf.json -*_override.tf -*_override.tf.json - -### Helm ### -# Chart dependencies -**/charts/*.tgz - - -# Ignore generated credentials from google-github-actions/auth -gha-creds-*.json - - -# Terraform -.terraformrc -*_override.tf -*_override.tf.json -*.tfstate -*.tfstate.* -*.tfvars -crash.*.log -crash.log -override.tf -override.tf.json -terraform.rc - -# Teraform Demo -.terraform -.terraform.lock.hcl - -# ============================================================================ -# Environment Management -# ============================================================================ - -### Version Management Tools ### -mise.toml - -### VSCode ### -*.code-workspace - -# ============================================================================ -# Project-Specific Files -# ============================================================================ -digests.txt -images.json - -# Binary files -fault-quarantine/fault-quarantine -fault-remediation/fault-remediation -health-events-analyzer/health-events-analyzer -health-monitors/syslog-health-monitor/syslog-health-monitor -health-monitors/kubernetes-object-monitor/kubernetes-object-monitor -labeler/labeler -metadata-collector/metadata-collector -node-drainer/node-drainer -platform-connectors/platform-connectors -event-exporter/event-exporter -tests/scale-tests/FQM_LATENCY_TEST_PLAN.md -tests/scale-tests/CONCURRENT_DRAIN_TEST_PLAN.md -tests/scale-tests/results/*.csv -tests/scale-tests/cmd/fqm-scale-test/results/ +# ============================================================================== +# Temporary Implementation Documents +# ============================================================================== +NVML_CONTAINERIZATION_IMPLEMENTATION.md +PR720_REFACTOR_PLAN.md +ARCH_CONTEXT.md +IMPLEMENTATION_TASKS.md + +# ============================================================================== +# Git Worktrees +# ============================================================================== +.worktrees/ diff --git a/.golangci.yml b/.golangci.yml index dd53c499b..84717c431 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -18,7 +18,8 @@ run: modules-download-mode: readonly issues-exit-code: 1 tests: false - allow-parallel-runners: false + allow-parallel-runners: true + output: formats: text: @@ -28,6 +29,7 @@ output: code-climate: path: code-quality-report.json path-prefix: "" + linters: enable: - asciicheck @@ -46,6 +48,7 @@ linters: - goconst - gocritic - gosec + - govet - lll - misspell - nestif @@ -53,7 +56,6 @@ linters: - noctx - whitespace - wsl_v5 - - gosec exclusions: generated: lax presets: @@ -65,8 +67,8 @@ linters: - third_party$ - builtin$ - examples$ -issues: - uniq-by-line: false + - code-generator/cmd$ + formatters: enable: - gofmt @@ -77,3 +79,7 @@ formatters: - third_party$ - builtin$ - examples$ + - code-generator/cmd$ + +issues: + uniq-by-line: false diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..dc8345242 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,447 @@ +# AGENTS.md - Persistent Context for AI Assistants + +> **Last Updated:** 2026-02-04 +> **Active Branch:** `feat/cloud-native-healthevents` ✅ READY +> **Base Commit:** `105cd6c` (Use cuda image from NVCR to avoid rate limits (#792)) +> **Head Commit:** `a7f3cc7` (fix(tests): register HealthEvent types and fix context keys) +> **New PR:** https://github.com/NVIDIA/NVSentinel/pull/795 (DRAFT) +> **Old PR:** https://github.com/NVIDIA/NVSentinel/pull/794 (superseded - incorrectly based) +> **Status:** ✅ Phase 4: Validation COMPLETE (controller flow verified) + +--- + +## Current Task: Rebase Cloud-Native Storage onto Main + +### Problem +The `feat/cloud-native-storage` branch was created from commit `59578f3`, which is on the +`device-api-server` lineage **after** commit `d6c5c46` that deleted all NVSentinel core code. + +The branch is missing: +- `commons/`, `data-models/`, `health-monitors/`, `labeler/`, `lint/` +- `remediations/`, `reports/`, `scalers/`, `sentinel/`, `services/` +- `CONTRIBUTING.md`, `.coderabbit.yaml`, `RELEASE.md`, `ROADMAP.md` +- Most GitHub Actions workflows + +### Solution +Cherry-pick our commits onto a fresh branch from `origin/main`. + +### Commits to Cherry-Pick (in order) +``` +7fc1f70 chore - Add small set of GitHub checks and copilot rules (#691) +a07681f feat: introduce k8s-idiomatic Go SDK for Device API (#692) +6a94259 chore: update documentation (#693) +8ad7b7a api: add ProviderService proto for device-api-server +b907fb1 feat: add device-api-server with NVML fallback provider +95cea2e fix(security): bind gRPC TCP listener to localhost by default +cf88679 docs(provider): clarify Heartbeat RPC is reserved for future use +ff88ab8 feat(consumer): populate ListMeta.ResourceVersion in ListGpus response +e37ed8c docs: document module structure and naming conventions +bd1b3ad fix: align module paths to github.com/nvidia/nvsentinel +cf1a193 refactor: consolidate to unified GpuService with standard CRUD methods +f7de28b fix(build): use Go 1.25 for container builds +fd4c919 fix(nvml-provider): parse command-line flags before returning config +cdbcfc3 fix(helm): remove invalid --provider-address flag from server args +18bb07c fix(helm): correct sidecar test values for cluster deployment +de7751e feat(demo): improve cross-platform builds and idempotency +28ef6be fix(demo): remove unreliable metrics check from verify_gpu_registration +cc2fcce fix(demo): use correct container name 'nvsentinel' instead of 'device-api-server' +18946e1 fix(ci): update protoc version to v33.4 to match generated files +a231cf3 docs: add hybrid device-apiserver design for PR #718 + #720 merge +59578f3 chore: add .worktrees/ and temp docs to .gitignore +3260812 feat: implement cloud-native GPU health event management +``` + +### Worktree Locations +- **Old (broken):** `/Users/eduardoa/src/github/nvidia/NVSentinel/.worktrees/cloud-native-storage` (to be removed) +- **New (active):** `/Users/eduardoa/src/github/nvidia/NVSentinel/.worktrees/cloud-native-healthevents` ✅ + +--- + +## Feature: Cloud-Native GPU Health Event Management + +### Architecture +``` +┌─────────────────┐ gRPC ┌──────────────────┐ +│ HealthProvider │ ─────────▶│ Device-API-Server│ +│ (DaemonSet) │ │ │ +└─────────────────┘ └────────┬─────────┘ + │ │ + │ NVML │ CRDPublisher + ▼ ▼ + ┌───────┐ ┌───────────────┐ + │ GPU │ │ HealthEvent │ + └───────┘ │ (CRD) │ + └───────┬───────┘ + │ + ┌─────────────────────────┼─────────────────────────┐ + │ │ │ + ▼ ▼ ▼ + ┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐ + │ QuarantineCtrl │───▶│ DrainCtrl │───▶│ RemediationCtrl │ + │ (cordon node) │ │ (evict pods) │ │ (reboot/reset) │ + └──────────────────┘ └──────────────────┘ └──────────────────┘ +``` + +### Phase Progression +``` +New → Quarantined → Drained → Remediated → Resolved +``` + +### Key Files Created +``` +api/nvsentinel/v1alpha1/ +├── doc.go +├── groupversion_info.go +├── healthevent_types.go +└── zz_generated.deepcopy.go + +pkg/controllers/healthevents/ +├── conditions.go +├── drain_controller.go +├── drain_controller_test.go +├── metrics.go +├── quarantine_controller.go +├── quarantine_controller_test.go +├── remediation_controller.go +├── remediation_controller_test.go +├── ttl_controller.go +└── ttl_controller_test.go + +pkg/deviceapiserver/crdpublisher/ +├── metrics.go +├── publisher.go +└── publisher_test.go + +pkg/healthprovider/ +├── nvml_source.go +├── nvml_source_stub.go +├── provider.go +└── provider_test.go + +cmd/controller-test/main.go +cmd/health-provider/main.go +cmd/health-provider/main_stub.go + +deployments/helm/nvsentinel/crds/nvsentinel.nvidia.com_healthevents.yaml +deployments/helm/nvsentinel/templates/policy-configmap.yaml +deployments/helm/health-provider/ +``` + +### Integration Test Results (2026-02-04) +- **Cluster:** AWS EKS with GPUs +- **Kubeconfig:** `/Users/eduardoa/.kube/config-aws-gpu` +- **Result:** SUCCESS - Full phase progression in 2 seconds +- **Node tested:** `ip-10-0-0-236` + +### Important Fixes Applied +1. **Predicate removal:** Controllers don't use `WithEventFilter` for phase filtering + (status subresource updates don't reliably trigger watch events) +2. **Empty phase handling:** `QuarantineController` treats `phase=""` as `phase=New` +3. **Field index:** Added `spec.nodeName` index for Pods in controller-test main.go + +--- + +## Commands Reference + +### Create new worktree from main +```bash +cd /Users/eduardoa/src/github/nvidia/NVSentinel +git fetch origin main +git worktree add .worktrees/cloud-native-healthevents origin/main -b feat/cloud-native-healthevents +``` + +### Cherry-pick all commits +```bash +cd .worktrees/cloud-native-healthevents +git cherry-pick 7fc1f70^..3260812 +``` + +### Run integration test +```bash +export KUBECONFIG=/Users/eduardoa/.kube/config-aws-gpu +./bin/controller-test --health-probe-bind-address=:18081 --metrics-bind-address=:18080 -v=2 +``` + +### Create test HealthEvent +```yaml +apiVersion: nvsentinel.nvidia.com/v1alpha1 +kind: HealthEvent +metadata: + name: test-full-flow +spec: + source: integration-test + nodeName: + componentClass: GPU + checkName: xid-error-check + isFatal: true + recommendedAction: RESTART_VM + detectedAt: "2026-02-04T16:00:00Z" +``` + +--- + +## Current Task: E2E Test Migration + +### Problem +Old E2E tests (in `tests/`) assume MongoDB + old microservices architecture: +- `fault_quarantine_test.go` - Tests fault-quarantine microservice +- `fault_remediation_test.go` - Tests fault-remediation microservice +- `node_drainer_test.go` - Tests node-drainer microservice +- `health_events_analyzer_test.go` - Tests MongoDB aggregation pipelines +- `smoke_test.go` - End-to-end flow with MongoDB + +New architecture uses HealthEvent CRD + Kubernetes controllers. Tests need **updating**, not rebuilding. + +### Decision: Update Existing Tests to Use CRD-Based Flow + +**Approach:** +1. Replace MongoDB assertions with HealthEvent CRD assertions +2. Replace microservice coordination checks with controller reconciliation checks +3. Leverage existing integration test harness (`cmd/controller-test/main.go`) +4. Preserve test scenarios (fault detection → quarantine → drain → remediation) + +### Migration Mapping + +| Old Test Pattern | New Test Pattern | +|------------------|------------------| +| Verify MongoDB change stream | Verify HealthEvent CRD creation | +| Check fault-quarantine processed | Check `status.phase = Quarantined` | +| Check node-drainer evicted pods | Check `status.phase = Drained` | +| Check fault-remediation completed | Check `status.phase = Remediated` | +| MongoDB collection cleanup | HealthEvent CRD deletion | + +### Implementation Plan (3-4 weeks) + +**Phase 1: Audit (2-3 days)** +- Catalog all E2E test scenarios +- Map old assertions to new CRD assertions +- Identify KWOK vs real cluster requirements + +**Phase 2: Infrastructure (3-5 days)** +- Update test setup to use HealthEvent CRD +- Create test fixtures/helpers for CRD creation +- Update cleanup logic + +**Phase 3: Migrate Tests (1-2 weeks)** +- Fault Detection → HealthEvent Creation +- Quarantine Flow → phase=Quarantined +- Drain Flow → phase=Drained +- Remediation Flow → phase=Remediated + +**Phase 4: Validation (3-5 days)** +- Run against KWOK/kind +- Validate against AWS EKS +- Performance validation + +### Key Test Files to Update +``` +tests/ +├── smoke_test.go # Full lifecycle test +├── fault_quarantine_test.go # Quarantine logic +├── fault_remediation_test.go # Remediation logic +├── node_drainer_test.go # Drain logic +├── health_events_analyzer_test.go # Pattern detection (may defer) +└── helpers/ + ├── healthevent.go # Update to use CRD client + ├── fault_quarantine.go # Update assertions + └── kube.go # Keep Kubernetes helpers +``` + +### Success Criteria +- [ ] All original scenarios migrated +- [ ] Tests pass consistently (≥95%) +- [ ] Test execution ≤5 min per scenario +- [ ] Full lifecycle coverage +- [ ] Zero MongoDB dependencies +- [ ] Runnable on KWOK/kind and AWS EKS + +--- + +## Phase 3 Progress: Test Migration + +### Completed +- [x] `smoke_test.go` - Full lifecycle tests migrated +- [x] `fault_quarantine_test.go` - QuarantineController tests migrated +- [x] `node_drainer_test.go` - DrainController tests migrated +- [x] `fault_remediation_test.go` - RemediationController tests migrated + +### fault_quarantine_test.go Migration Summary + +**Migrated tests:** +- `TestNonFatalEventDoesNotTriggerQuarantine` - Verify isFatal=false skips quarantine +- `TestHealthyEventDoesNotTriggerQuarantine` - Verify isHealthy=true skips quarantine +- `TestPreCordonedNodeHandling` - Pre-cordoned nodes handled correctly +- `TestQuarantineSkipOverride` - `spec.overrides.quarantine.skip` works +- `TestMultipleEventsOnSameNode` - Multiple events on same node + +**Removed (old microservice-specific):** +- `TestDontCordonIfEventDoesntMatchCELExpression` - CEL filtering (fault-quarantine specific) +- `TestCircuitBreakerCursorCreateSkipsAccumulatedEvents` - Circuit breaker (MongoDB-specific) +- `TestFaultQuarantineWithProcessingStrategy` - Processing strategy (data-models specific) + +### smoke_test.go Migration Summary + +**TestFatalHealthEvent** - Full remediation flow +``` +New → Quarantined → Draining → Drained → Remediated → Resolved +``` +- Removed: HTTP SendHealthEventsToNodes(), node label sequence watching, statemanager dependency +- Added: CreateHealthEventCRD(), WaitForHealthEventPhase(), phase-based assertions +- Kept: Node cordon/uncordon, RebootNode CR, log-collector job assertions + +**TestFatalUnsupportedHealthEvent** - CONTACT_SUPPORT events +- Tests events that skip automatic remediation (XID 145) +- Asserts event stays in Drained phase (never reaches Remediated) +- Verifies no log-collector job created + +### node_drainer_test.go Migration Summary + +**Migrated tests:** +- `TestDrainControllerBasicFlow` - Basic drain via HealthEvent phases +- `TestDrainSkipOverride` - `spec.overrides.drain.skip` works +- `TestDrainWithKubeSystemExclusion` - kube-system pods not evicted +- `TestDrainPhaseSequence` - Full phase sequence validation + +**Removed (old microservice-specific):** +- `TestNodeDrainerEvictionModes` - ConfigMap-based eviction mode config +- `TestNodeDrainerPartialDrain` - Partial drain with entitiesImpacted + +### fault_remediation_test.go Migration Summary + +**Migrated tests:** +- `TestRemediationControllerBasicFlow` - Basic remediation via phases +- `TestMultipleRemediationsOnSameNode` - Multiple CRs on same node +- `TestContactSupportDoesNotTriggerRemediation` - CONTACT_SUPPORT skips remediation +- `TestFullPhaseSequenceToResolved` - Full lifecycle New → Resolved + +### Deferred +- [ ] `health_events_analyzer_test.go` - (MongoDB-specific, pattern detection) + +--- + +## Phase 4 Results: Validation (COMPLETE ✅) + +### Manual Validation (2026-02-04) +- **Cluster:** `kubernetes-admin@holodeck-cluster` (AWS EKS) +- **Test Node:** `ip-10-0-0-10` +- **Result:** SUCCESS + +**Validation Steps:** +1. Built `controller-test` binary +2. Started controllers locally (`./bin/controller-test`) +3. Created HealthEvent CRD manually +4. Verified phase transitions: `New → Quarantined → Draining` +5. Verified node cordoned (`spec.unschedulable=true`) +6. Verified `NodeQuarantined` condition set + +**Test Infrastructure Fixes Applied:** +- Registered `nvsentinelv1alpha1.HealthEvent` types in test scheme +- Moved `keyHealthEventName` context key to `main_test.go` +- Removed unused `v1` import from `smoke_test.go` + +**Note:** Full E2E test suite requires KWOK nodes for `amd64_group` tests. +The `arm64_group` tests (smoke tests) use real nodes but require longer timeout. + +--- + +## Phase 2 Results: Test Infrastructure (COMPLETE ✅) + +Created `tests/helpers/healthevent_crd.go` with 23 helper functions. + +### Builder Pattern +```go +event := NewHealthEventCRD("node-1"). + WithFatal(true). + WithCheckName("GpuXidError"). + WithErrorCodes("79"). + WithRecommendedAction(nvsentinelv1alpha1.ActionRestartVM). + Build() +``` + +### CRD Operations +- `CreateHealthEventCRD(ctx, t, c, event)` - Create CRD +- `GetHealthEventCRD(ctx, c, name)` - Get by name +- `DeleteHealthEventCRD(ctx, t, c, name)` - Delete +- `ListHealthEventCRDs(ctx, c)` - List all +- `ListHealthEventCRDsForNode(ctx, c, nodeName)` - Filter by node +- `DeleteAllHealthEventCRDs(ctx, t, c)` - Cleanup all + +### Phase Waiting +- `WaitForHealthEventPhase(ctx, t, c, name, phase)` - Wait for phase +- `WaitForHealthEventPhaseNotEqual(ctx, t, c, name, phase)` - Wait to leave phase +- `WaitForHealthEventCondition(ctx, t, c, name, condType, status)` - Wait for condition +- `WaitForHealthEventPhaseSequence(ctx, t, c, name, []phases)` - Wait for sequence + +### Assertions +- `AssertHealthEventPhase(t, event, phase)` - Assert phase +- `AssertHealthEventNotExists(ctx, t, c, name)` - Assert doesn't exist +- `AssertHealthEventHasCondition(t, event, condType, status)` - Assert condition +- `AssertHealthEventNeverReachesPhase(ctx, t, c, name, phase)` - Assert never reaches +- `AssertNodeQuarantinedCondition(t, event)` - Assert quarantine condition +- `AssertPodsDrainedCondition(t, event)` - Assert drain condition +- `AssertRemediatedCondition(t, event)` - Assert remediation condition +- `AssertResolvedAtSet(t, event)` - Assert resolved timestamp + +### Migration Helpers (Old HTTP → New CRD) +- `SendHealthEventViaCRD(ctx, t, c, template)` - Drop-in for `SendHealthEvent` +- `SendHealthyEventViaCRD(ctx, t, c, nodeName)` - Drop-in for `SendHealthyEvent` +- `TriggerFullRemediationFlowViaCRD(ctx, t, c, nodeName)` - Trigger full flow + +--- + +## Phase 1 Audit Results (2026-02-04) + +### Test Files Audited + +| File | Tests | Key Scenarios | +|------|-------|---------------| +| `smoke_test.go` | 2 | Full fatal event flow, unsupported event flow | +| `fault_quarantine_test.go` | 4 | CEL filtering, pre-cordoned nodes, circuit breaker, processing strategy | +| `node_drainer_test.go` | 2 | Eviction modes, partial drain | +| `fault_remediation_test.go` | 1 | RebootNode CR creation after remediation | + +### Common Old → New Assertion Mapping + +| Old Pattern | New Pattern | +|-------------|-------------| +| `SendHealthEvent()` HTTP POST | Create HealthEvent CRD via K8s client | +| MongoDB change stream watch | HealthEvent CRD informer/watch | +| Node label `nvsentinel-state=draining` | HealthEvent `status.phase=Draining` | +| Node label `nvsentinel-state=drain-succeeded` | HealthEvent `status.phase=Drained` | +| `CheckNodeEventExists("NodeDraining")` | HealthEvent `status.conditions` check | +| `WaitForRebootNodeCR()` | HealthEvent `status.phase=Remediated` | + +### Required New Helper Functions + +```go +// CRD Operations +CreateHealthEventCRD(ctx, client, spec) error +GetHealthEventCRD(ctx, client, name) (*HealthEvent, error) +DeleteHealthEventCRD(ctx, client, name) error +ListHealthEventCRDs(ctx, client, opts) ([]HealthEvent, error) + +// Phase Waiting +WaitForHealthEventPhase(ctx, client, name, phase, timeout) error +WaitForHealthEventCondition(ctx, client, name, condType, status) error + +// Assertions +AssertHealthEventPhase(t, event, expectedPhase) +AssertHealthEventNotExists(ctx, t, client, name) +AssertHealthEventHasCondition(t, event, condType, status) +``` + +### Test Migration Priority + +1. **High**: `smoke_test.go` - Core E2E flow, validates full lifecycle +2. **High**: `fault_quarantine_test.go` - QuarantineController validation +3. **Medium**: `node_drainer_test.go` - DrainController validation +4. **Medium**: `fault_remediation_test.go` - RemediationController validation +5. **Low**: `health_events_analyzer_test.go` - Pattern detection (defer, MongoDB-specific) + +--- + +## Related PRs and Issues +- PR #795: Cloud-native health events (current work) +- PR #794: Superseded (incorrectly based branch) +- PR #718, #720: Original device-api-server proposals being enhanced +- Design doc: `docs/plans/2026-02-04-hybrid-device-apiserver-design.md` diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index aa43cb83d..e7e16ac6f 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -1,1335 +1,130 @@ -# NVSentinel Development Guide +# NVIDIA Device API: Development Guide -This guide covers project structure, development workflows, and best practices for contributing to NVSentinel. +This guide covers the development setup and workflows for contributing to the NVIDIA Device API. -## 📋 Table of Contents +## Module Structure -- [Getting Started](#-getting-started) -- [Project Architecture](#-project-architecture) -- [Development Environment Setup](#-development-environment-setup) -- [Development Workflows](#-development-workflows) -- [Module Development](#-module-development) -- [Testing](#-testing) -- [Code Standards](#-code-standards) -- [CI/CD Pipeline](#-cicd-pipeline) -- [Debugging](#-debugging) -- [Makefile Reference](#-makefile-reference) +This repository is a multi-module monorepo containing multiple Go modules: -## 🚀 Getting Started +| Module | Path | Description | +|--------|------|-------------| +| `github.com/nvidia/nvsentinel` | `/` | Device API Server implementation | +| `github.com/nvidia/nvsentinel/api` | `/api` | API definitions (protobuf and Go types) | +| `github.com/nvidia/nvsentinel/client-go` | `/client-go` | Kubernetes-style gRPC clients | +| `github.com/nvidia/nvsentinel/code-generator` | `/code-generator` | Code generation tools | -### Quick Setup +The API module is designed to be imported independently by consumers who only need the type definitions. -```bash -git clone https://github.com/nvidia/nvsentinel.git -cd nvsentinel -make dev-env-setup # Install all dependencies -``` - -### Tool Version Management - -NVSentinel uses `.versions.yaml` for centralized version management across: -- Local development -- CI/CD pipelines -- Container builds - -**View current versions**: -```bash -make show-versions -``` - -### Prerequisites - -**Core Tools** (required): -- [Go 1.25+](https://golang.org/dl/) - See `.versions.yaml` for exact version -- [Docker](https://docs.docker.com/get-docker/) -- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) -- [Helm 3.0+](https://helm.sh/docs/intro/install/) -- [Protocol Buffers Compiler](https://grpc.io/docs/protoc-installation/) -- [yq](https://github.com/mikefarah/yq) - YAML processor for version management - -**Development Tools**: -- [golangci-lint](https://golangci-lint.run/usage/install/) -- [gotestsum](https://github.com/gotestyourself/gotestsum) -- [gocover-cobertura](https://github.com/boumenot/gocover-cobertura) -- [addlicense](https://github.com/google/addlicense) -- [Poetry](https://python-poetry.org/) -- [shellcheck](https://github.com/koalaman/shellcheck) - -**Optional** (for local Kubernetes development): -- [Tilt](https://tilt.dev/) -- [ctlptl](https://github.com/tilt-dev/ctlptl) -- [Kind](https://kind.sigs.k8s.io/) -- [MongoDB Compass](https://www.mongodb.com/products/compass) - -**Quick install** (installs and configures all tools): -```bash -make dev-env-setup -``` - -This will: -- Detect your OS (Linux/macOS) and architecture -- Install yq and check for required tools -- Install development and Go tools -- Configure Python gRPC tools - -**Automated installation**: To skip interactive prompts and auto-install all dependencies: -```bash -make dev-env-setup AUTO_MODE=true -``` - -**Debugging setup issues**: If the setup script fails, enable debug mode for detailed output: -```bash -DEBUG=true make dev-env-setup AUTO_MODE=true -``` - -This will show: -- Architecture detection and mappings -- URL construction for all downloads -- HTTP response codes for failed downloads -- Detailed error messages with suggestions +## Architecture -Common setup issues and solutions are documented in the [Debugging](#-debugging) section. +This project bridges **gRPC** (for node-local performance) with **Kubernetes API Machinery** (for developer experience). -### Build System +1. **Definitions**: `api/proto` (Wire format) and `api/device` (Go types). +2. **Conversion**: `api/device/${version}/converter.go` maps gRPC messages to K8s-style structs. +3. **Generation**: A pipeline driven by `code-generator/kube_codegen.sh`, which utilizes a modified `client-gen` to produce gRPC-backed Kubernetes clients in the `client-go` module. -**Unified build system** features: -- **Consistent interface**: All modules support common targets (`all`, `lint-test`, `clean`) -- **Technology-aware**: Appropriate tooling for Go, Python, and shell scripts -- **Delegation pattern**: Top-level Makefiles delegate to individual modules -- **Repo-root context**: Docker builds use consistent paths -- **Multi-platform support**: Built-in `linux/arm64,linux/amd64` via Docker buildx - -## 🏗️ Project Architecture - -NVSentinel follows a **microservices architecture** with event-driven communication: - -### Core Principles +--- -1. **Independence**: Modules operate autonomously -2. **Event-Driven**: Communication through MongoDB change streams -3. **Modular**: Pluggable health monitors -4. **Cloud-Native**: Kubernetes-first design +## Code Generation Pipeline -### Module Types - -``` -nvsentinel/ -├── health-monitors/ # Hardware/software fault detection -│ ├── gpu-health-monitor/ # Python - DCGM GPU monitoring -│ ├── syslog-health-monitor/ # Go - System log monitoring -│ └── csp-health-monitor/ # Go - Cloud provider monitoring -├── platform-connectors/ # gRPC event ingestion service -├── fault-quarantine/ # CEL-based event quarantine logic -├── fault-remediation/ # Kubernetes controller for remediation -├── health-events-analyzer/ # Event analysis and correlation -├── health-event-client/ # Event streaming client -├── labeler/ # Node labeling controller -├── node-drainer/ # Graceful workload eviction -├── store-client/ # MongoDB interaction library (tested in CI) -└── log-collector/ # Log aggregation (shell scripts) -``` - -### Communication Flow +The NVIDIA Device API uses a multi-stage pipeline to bridge gRPC with Kubernetes API machinery. For module-specific details, see the [client-go Development Guide](./client-go/DEVELOPMENT.md). ```mermaid -sequenceDiagram - participant HM as Health Monitor - participant PC as Platform Connectors - participant DB as MongoDB - participant FM as Fault Module - - HM->>PC: gRPC health event - PC->>DB: Store event - DB->>FM: Change stream notification - FM->>DB: Query related events - FM->>K8s: Execute remediation action -``` - -## 🛠️ Development Environment Setup - -### 1. Local Development with Tilt - -Tilt provides the fastest development experience with hot reloading. - -```bash -# Quick start - create cluster and start Tilt in one command -make dev-env # Creates cluster and starts Tilt - -# Manual step-by-step approach -make cluster-create # Creates ctlptl-managed Kind cluster with registry -make tilt-up # Starts Tilt with UI (runs: tilt up -f tilt/Tiltfile) - -# Check status -make cluster-status # Check cluster and registry status - -# View Tilt UI -# Navigate to http://localhost:10350 - -# Stop everything when done -make dev-env-clean # Stops Tilt and deletes cluster - -# Or stop individually -make tilt-down # Stops Tilt (runs: tilt down -f tilt/Tiltfile) -make cluster-delete # Deletes the cluster -``` - -**ctlptl Cluster Features:** - -- Declarative cluster configuration with YAML -- Multi-node Kind cluster (3 control-plane, 2 worker nodes) -- Cluster name: `kind-nvsentinel` (required `kind-` prefix) -- Integrated local container registry at `localhost:5001` -- Automatic registry configuration for Tilt -- Simplified cluster lifecycle management -- No external dependencies beyond Docker, ctlptl, and Kind - -### 2. Manual Development Setup - -For module-specific development without full cluster: - -```bash -# Set up Go environment -export GOPATH=$(go env GOPATH) -export GO_CACHE_DIR=$(go env GOCACHE) - -# Install development dependencies -go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@latest -go install gotest.tools/gotestsum@latest -go install github.com/boumenot/gocover-cobertura@latest - -# For controller modules -go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest -``` - -### 3. Go Module Configuration - -Go module dependencies are handled automatically: - -```bash -# Dependencies managed via go.mod files with replace directives for local development -# No manual GOPRIVATE configuration needed -# Private repository authentication handled via SSH keys -``` - -## 🔄 Development Workflows - -### Daily Development Workflow - -1. **Start Development Session** - ```bash - git checkout main - git pull origin main - git checkout -b feature/your-feature-name - - # Start local development environment - make dev-env # Creates ctlptl-managed cluster and starts Tilt - ``` - -2. **Develop with Live Reload** - ```bash - # Edit code - Tilt automatically rebuilds and redeploys - vim health-monitors/syslog-health-monitor/pkg/monitor/monitor.go - - # View logs in Tilt UI at http://localhost:10350 - # Or use kubectl for specific logs (note: syslog-health-monitor runs as DaemonSet with -regular and -kata variants) - kubectl logs -f daemonset/nvsentinel-syslog-health-monitor-regular -n nvsentinel - ``` - -3. **Test Changes** - ```bash - # Run tests locally (while Tilt is running) - make health-monitors-lint-test-all # All health monitors - make health-events-analyzer-lint-test # Specific Go module - make platform-connectors-lint-test # Another Go module - - # Or run individual module tests directly (using standardized targets) - make -C health-monitors/syslog-health-monitor lint-test - make -C platform-connectors lint-test - make -C health-events-analyzer lint-test - - # Test integration with other services via Tilt UI - # Access services via port-forwards set up by Tilt - ``` - -4. **Validate Before Commit** - ```bash - # Run full test suite - make lint-test-all - - # Stop Tilt for final testing if needed - make tilt-down - ``` - -5. **Commit and Push** - ```bash - git add . - git commit -s -m "feat: add new monitoring capability" - git push origin feature/your-feature-name - - # Clean up development environment - make dev-env-clean - ``` - -### Protocol Buffer Development - -When modifying `.proto` files: - -```bash -# Generate protobuf files -make protos-lint - -# This runs: -# - protoc generation for Go modules -# - Python protobuf generation for GPU monitor -# - Import path fixes for Python -# - Git diff check to ensure files are up to date -``` - -### Container Development - -The project provides a **unified Docker build system** with consistent patterns across all modules. All builds support multi-platform architecture, build caching, and proper context management. - -#### Environment Variables - -Set these for production-like builds: - -```bash -# Docker configuration (standardized across all modules) -export CONTAINER_REGISTRY="ghcr.io" -export CONTAINER_ORG="your-github-username" # Defaults to repository owner -export CI_COMMIT_REF_NAME="feature-branch" # Or your branch name - -# These are computed automatically by common.mk: -# SAFE_REF_NAME=$(echo $CI_COMMIT_REF_NAME | sed 's/\//-/g') -# PLATFORMS="linux/arm64,linux/amd64" -# MODULE_NAME=$(basename $(CURDIR)) -``` - -#### Docker Build Commands - -**Build System Overview** - -The Docker build system uses shared patterns via `common.mk` for Go modules, with specialized handling for Python and container-only modules. Each module maintains its own Docker configuration. - -**Main build targets (delegated to individual modules):** +graph TD + API["API Definitions
(nvidia/nvsentinel/api)"] -->|Input| CG(client-gen
*Custom Build*) + API -->|Input| LG(lister-gen) -```bash -# Local Development (--load) - builds images into local Docker daemon -make docker-all # All images locally (delegates to docker/Makefile) -make docker-health-monitors # All health monitor images locally -make docker-main-modules # All non-health-monitor images locally - -# CI/Production (--push) - builds and pushes directly to registry -make docker-publish-all # Build and push all images to registry -make docker-publish-health-monitors # Build and push health monitor images -make docker-publish-main-modules # Build and push main module images - -# Individual module targets (via common.mk) -make docker-syslog-health-monitor # Build syslog health monitor locally -make docker-publish-syslog-health-monitor # Build and push to registry -make docker-platform-connectors # Build platform connectors locally -make docker-publish-platform-connectors # Build and push to registry - -# Special cases -make docker-gpu-health-monitor # Both DCGM 3.x and 4.x versions locally -make docker-log-collector # Container-only module (shell + Python) -``` - -**Direct docker/ Makefile usage:** - -```bash -cd docker - -# Local development builds (--load) -make build-all # Build all 12 images locally -make build-health-monitors # Build health monitor group locally -make build-syslog-health-monitor # Build specific module locally - -# CI/production builds (--push) -make publish-all # Build and push all images to registry -make publish-syslog-health-monitor # Build and push specific image to registry - -# Utility commands -make setup-buildx # Setup multi-platform builder -make clean # Remove all nvsentinel images -make list # List built nvsentinel images -make help # Show all available targets -``` - -**Individual module usage:** - -```bash -# Go modules (common.mk patterns) -make -C health-monitors/syslog-health-monitor docker-build # Local build with remote cache -make -C health-monitors/syslog-health-monitor docker-build-local # Local build, no remote cache (faster) -make -C health-monitors/syslog-health-monitor docker-publish # CI build -make -C platform-connectors docker-build # Local build with remote cache -make -C platform-connectors docker-build-local # Local build, no remote cache (faster) -make -C platform-connectors docker-publish # CI build -make -C health-events-analyzer docker-build-local # Local build, no remote cache (faster) -make -C health-events-analyzer docker-publish # CI build - -# Python module (specialized patterns) -make -C health-monitors/gpu-health-monitor docker-build-dcgm3 # DCGM 3.x local -make -C health-monitors/gpu-health-monitor docker-publish-dcgm4 # DCGM 4.x CI - -# Container-only module (shell + Python) -make -C log-collector docker-build-log-collector # Local build -make -C log-collector docker-publish-log-collector # CI build -``` - -#### Module-Level Docker Builds - -Each module provides Docker targets with common patterns: - -```bash -# Go modules (common.mk patterns) -make -C health-monitors/syslog-health-monitor docker-build # Local with remote cache -make -C health-monitors/syslog-health-monitor docker-build-local # Local, no remote cache (recommended) -make -C health-monitors/syslog-health-monitor docker-publish # CI/production -make -C platform-connectors docker-build # Local with remote cache -make -C platform-connectors docker-build-local # Local, no remote cache (recommended) -make -C platform-connectors docker-publish # CI/production - -# Python module (specialized patterns) -make -C health-monitors/gpu-health-monitor docker-build-dcgm3 # DCGM 3.x local -make -C health-monitors/gpu-health-monitor docker-build-dcgm4 # DCGM 4.x local -make -C health-monitors/gpu-health-monitor docker-publish # Push both versions - -# Container-only module (shell + Python) -make -C log-collector docker-build # Both log-collector and file-server-cleanup -make -C log-collector docker-publish # Push both components - -# Legacy compatibility (all modules) -make -C [module] image # Calls docker-build -make -C [module] publish # Calls docker-publish -``` - -#### Docker Build Features - -All builds support consistent features: - -1. **Multi-Platform Support**: `linux/arm64,linux/amd64` via `common.mk` (`docker-build`); `linux/amd64` for `docker-build-local` -2. **Build Caching**: Registry-based build cache for faster builds (`docker-build`); local cache only (`docker-build-local`) -3. **Repo-Root Context**: All builds use consistent repo-root context -4. **Dynamic Tagging**: Uses branch/tag name (`${SAFE_REF_NAME}`) for `docker-build`; simple `module:local` for `docker-build-local` -5. **Registry Integration**: NVCR.io registry paths for `docker-build` and `docker-publish` -6. **Module Auto-Detection**: Automatic module name detection via `$(MODULE_NAME)` - -**For Local Development**, use `docker-build-local` to avoid registry authentication issues and build faster. - -#### Example Build Scenarios - -**Local Development:** -```bash -# Recommended: Fast local build (single platform, no remote cache) -make -C health-monitors/syslog-health-monitor docker-build-local - -# Alternative: Local build with remote cache (multi-platform, slower) -make -C health-monitors/syslog-health-monitor docker-build - -# Legacy: Quick local build -make -C health-monitors/syslog-health-monitor image -``` - -**CI-like Build:** -```bash -**CI-like Build:** -```bash -# Set up environment like GitHub Actions -export CONTAINER_REGISTRY="ghcr.io" -export CONTAINER_ORG="your-github-username" -export CI_COMMIT_REF_NAME="main" - -# Build all images with full CI features (standardized) -make docker-all - -# Images will be tagged like: -# ghcr.io/your-github-username/syslog-health-monitor:main -# ghcr.io/nvidia/nvsentinel/gpu-health-monitor:main-dcgm-3.x -# ghcr.io/nvidia/nvsentinel/gpu-health-monitor:main-dcgm-4.x -``` - -**Testing Specific Module:** -```bash -# Recommended: Build and test individual module (fast, local) -make -C platform-connectors docker-build-local -docker run --rm platform-connectors:local --help + CG -->|Generates| CLIENT[client/versioned] + LG -->|Generates| LISTERS[listers/] -# Alternative: Build with full CI features -make docker-platform-connectors -docker run --rm ghcr.io/nvidia/nvsentinel/platform-connectors:fix-make-file-targets --help + CLIENT & LISTERS -->|Input| IG(informer-gen) + IG -->|Generates| INFORMERS[informers/] -# Build private repo module (fast, local) -make -C health-events-analyzer docker-build-local + CLIENT & LISTERS & INFORMERS -->|Final Output| SDK[Ready-to-use SDK] ``` -#### Build Cache Benefits - -The new system uses Docker BuildKit registry cache: -- **First build**: Downloads and caches layers -- **Subsequent builds**: Reuses cached layers for 10x+ speed improvement -- **Multi-developer**: Cache shared across team via registry - -#### Troubleshooting Docker Builds - -**Build failures:** -```bash -# Check buildx setup -make -C docker setup-buildx - -# Clean and retry -make -C docker clean -docker system prune -f -make docker-syslog-health-monitor -``` - -**Private repo access:** -```bash -# Verify SSH key access -git ls-remote git@github.com:dgxcloud/mk8s/some-private-repo.git - -# Build with debug output -BUILDKIT_PROGRESS=plain make docker-csp-health-monitor -``` - -**Registry issues:** -```bash -# Test registry login -docker login nvcr.io -u '$oauthtoken' -p "$NGC_PASSWORD" - -# Check image tags -make -C docker list -``` - -#### macOS/Docker Desktop Socket Directory Requirements - -**Problem:** On macOS with Docker Desktop, Unix domain sockets require the `/var/run` directory to exist inside containers, but this directory is not created by default in minimal container images. - -**Symptoms:** -- Services fail to start with errors like: `failed to listen on unix socket /var/run/nvsentinel.sock: no such file or directory` -- Tilt-based tests fail on macOS but pass on Linux -- gRPC Unix socket connections fail - -**Solution:** The project includes a Tilt-specific Helm values file that creates the `/var/run` directory using an initContainer: - -```yaml -# File: distros/kubernetes/nvsentinel/values-tilt-socket.yaml -# -# This values file is automatically included when running Tilt on macOS/Docker Desktop. -# It adds an initContainer to create /var/run directory for Unix socket communication. - -global: - initContainers: - - name: create-run-dir - image: busybox:latest - command: ['sh', '-c', 'mkdir -p /var/run'] - volumeMounts: - - name: socket-dir - mountPath: /var/run -``` - -**How it works:** -1. The `tilt/Tiltfile` automatically includes `values-tilt-socket.yaml` for local development -2. The initContainer runs before each service starts and creates the `/var/run` directory -3. Services can then create Unix sockets at `/var/run/nvsentinel.sock` -4. The socket directory is shared via an `emptyDir` volume mount - -**Platform-specific behavior:** -- **macOS/Docker Desktop:** Requires the initContainer workaround (automatically applied in Tilt) -- **Linux:** The `/var/run` directory typically exists in the container runtime environment -- **Production/Kubernetes:** Uses standard Helm values without the initContainer (not needed) - -**Note:** This is a development-only workaround for local macOS environments. Production deployments on Linux do not require this configuration. - -## 🧩 Module Development - -### Creating a New Health Monitor - -1. **Create Module Structure** - ```bash - mkdir -p health-monitors/my-monitor/{cmd,pkg,internal} - cd health-monitors/my-monitor - ``` - -2. **Initialize Go Module** - ```bash - go mod init github.com/nvidia/nvsentinel/health-monitors/my-monitor - ``` - -3. **Create Module Makefile** - ```bash - # Copy template from existing health monitor - cp ../syslog-health-monitor/Makefile ./Makefile - - # Update module-specific settings - sed -i 's/syslog-health-monitor/my-monitor/g' Makefile - sed -i 's/Syslog Health Monitor/My Monitor/g' Makefile - ``` - -4. **Implement gRPC Client** - ```go - // pkg/monitor/monitor.go - package monitor - - import ( - "context" - pb "github.com/nvidia/nvsentinel/platform-connectors/pkg/protos" - "google.golang.org/grpc" - ) - - type Monitor struct { - client pb.PlatformConnectorClient - } - - func (m *Monitor) SendEvent(ctx context.Context, event *pb.HealthEvent) error { - _, err := m.client.SendHealthEvent(ctx, event) - return err - } - ``` - -5. **Update health-monitors/Makefile** - ```bash - # Add your module to the health monitors list - # Edit health-monitors/Makefile: - # - Add 'my-monitor' to GO_HEALTH_MONITORS list - # - Add lint-test delegation target - # - Add build delegation target - # - Add clean delegation target - ``` - -6. **Test Your Module** - ```bash - # Test the individual module - make -C health-monitors/my-monitor lint-test - - # Test via health-monitors coordination - make -C health-monitors lint-test-my-monitor - - # Test via main Makefile delegation - make health-monitors-lint-test-all - ``` - -7. **Add to CI Pipeline** - The module will automatically be included in GitHub Actions workflows due to the standardized patterns. - -### Creating a New Core Module - -1. **Follow Kubernetes Controller Pattern** - ```bash - # Use controller-runtime for Kubernetes controllers - go get sigs.k8s.io/controller-runtime - ``` - -2. **Implement MongoDB Change Streams** - ```go - // Use store-client for MongoDB operations - import "github.com/nvidia/nvsentinel/store-client/pkg/client" - ``` - -3. **Add Proper RBAC** - Create Kubernetes RBAC manifests in `distros/kubernetes/nvsentinel/templates/`. - -## 🧪 Testing +### Build Sequence -### Test Strategy +When you run `make code-gen` from the root, the following sequence is executed: -- **Unit Tests**: Test individual functions and methods -- **Integration Tests**: Test module interactions -- **End-to-End Tests**: Test complete workflows via CI +1. **Protoc**: Compiles `.proto` into Go gRPC stubs in `api/gen/`. +2. **DeepCopy**: Generates `runtime.Object` methods required for K8s compatibility. +3. **Goverter**: Generates type conversion logic between Protobuf and Go structs. +4. **Custom client-gen**: Orchestrated by `code-generator/kube_codegen.sh` to produce the versioned Clientset, Informers, and Listers in `client-go/`. -### Running Tests - -The unified Makefile structure provides consistent testing across all modules: - -```bash -# Test all modules (delegates to all sub-Makefiles) -make lint-test-all # Main Makefile - runs everything - -# Test by category -make health-monitors-lint-test-all # All health monitors -make go-lint-test-all # All Go modules (common.mk patterns) +--- -# Test individual modules via delegation (main Makefile) -make health-events-analyzer-lint-test # Go module -make platform-connectors-lint-test # Go module -make store-client-lint-test # Go module -make log-collector-lint-test # Container module +## Development Workflow -# Test individual modules directly (common.mk patterns) -make -C health-monitors/syslog-health-monitor lint-test # Go module -make -C platform-connectors lint-test # Go module -make -C health-events-analyzer lint-test # Go module -make -C health-monitors/gpu-health-monitor lint-test # Python module +1. **Modify**: Edit the Protobuf definitions in `api/proto` or Go types in `api/device`. +2. **Update**: Update the conversion logic in `api/device/${version}/converter.go` to handle changes, if necessary. +3. **Generate**: Run `make code-gen` from the root. This updates the gRPC stubs, helper methods, and the `client-go` SDK. +4. **Verify**: Run `make verify-codegen` to ensure the workspace is consistent. +5. **Test**: Add tests to the affected module and run `make test` from the root. -# Use individual targets for development (common.mk) -cd health-monitors/syslog-health-monitor -make vet # Just go vet -make lint # Just golangci-lint -make test # Just tests -make coverage # Tests + coverage -make build # Build module -make binary # Build main binary +> [!NOTE] Use the fake clients in `client-go/client/versioned/fake` for testing controllers without a real gRPC server. -# Run specific test with verbose output -cd platform-connectors -go test -v ./pkg/connectors/... -``` +--- -### Test Requirements +## Code Standards & Compliance -Each module must include: -- Unit tests with `_test.go` suffix -- Coverage reporting via `go test -coverprofile` -- Integration tests where applicable -- Mocks for external dependencies +### Commit Messages & Signing (DCO) -### Python Testing (GPU Health Monitor) +We follow the [Conventional Commits](https://www.conventionalcommits.org) specification. Additionally, all commits **must** be signed off to comply with the Developer Certificate of Origin (DCO). ```bash -# Using the module's Makefile (recommended) -make -C health-monitors/gpu-health-monitor lint-test # Full lint-test -make -C health-monitors/gpu-health-monitor setup # Just Poetry setup -make -C health-monitors/gpu-health-monitor lint # Just Black check -make -C health-monitors/gpu-health-monitor test # Just tests -make -C health-monitors/gpu-health-monitor format # Run Black formatter - -# Manual Poetry commands -cd health-monitors/gpu-health-monitor -poetry install -poetry run pytest -v -poetry run black --check . -poetry run coverage run --source=gpu_health_monitor -m pytest +# Example: feat, fix, docs, chore, refactor +git commit -s -m "feat: add new GPU condition type" ``` -## 📏 Code Standards - -### Go Standards - -- **Linting**: Use `golangci-lint` with project configuration -- **Formatting**: Use `gofmt` (enforced by linting) -- **Imports**: Group standard, third-party, and local imports -- **Error Handling**: Always check and handle errors appropriately -- **Context**: Pass `context.Context` for cancellation and timeouts - -### Code Review Checklist - -- [ ] All tests pass -- [ ] Code coverage maintained or improved -- [ ] No linting violations -- [ ] Proper error handling -- [ ] Documentation updated -- [ ] License headers present -- [ ] Signed commits (`git commit -s`) - ### License Headers -All source files must include the Apache 2.0 license header: - -```bash -# Add license headers to new files -addlicense -f .github/headers/LICENSE . - -# Check license headers -make license-headers-lint -``` - -## 🚀 CI/CD Pipeline - -### GitHub Actions Workflows - -The project uses GitHub Actions for continuous integration with the following workflows: - -1. **`lint-test.yml`**: Code quality and testing - - Runs `lint-test` on all modules using standardized Makefile targets - - Includes health monitors, Go modules, Python linting, shell script validation - - Uses matrix strategy for parallel execution across components - -2. **`container-build-test.yml`**: Container build validation - - Validates Docker builds for all modules can complete successfully - - Uses the standardized `docker-build` targets from individual modules - - Runs on pull requests affecting container-related files - -3. **`e2e-test.yml`**: End-to-end testing - - Sets up Kind cluster with ctlptl for full integration testing - - Uses Tilt for deployment and testing - -4. **`publish.yml`**: Container image publishing -5. **`release.yml`**: Semantic release automation - -### Local CI Simulation - -```bash -# Run the same commands as GitHub Actions locally -make lint-test-all # Matches lint-test.yml workflow - -# Individual module CI commands (common.mk patterns) -make -C health-monitors/syslog-health-monitor lint-test -make -C health-monitors/gpu-health-monitor lint-test -make -C platform-connectors lint-test # Uses common.mk patterns -make -C log-collector lint-test # Shell + Python linting - -# Container builds (matches container-build-test.yml) -make -C health-monitors/syslog-health-monitor docker-build -make -C platform-connectors docker-build - -# Or run individual steps for debugging (common.mk targets) -cd health-monitors/syslog-health-monitor -make vet # go vet ./... -make lint # golangci-lint run -make test # gotestsum with coverage -make coverage # generate coverage reports - -# Manual commands (what common.mk executes) -go vet ./... -golangci-lint run --config ../.golangci.yml # Output format configured in .golangci.yml v2 -gotestsum --junitfile report.xml -- -race $(go list ./...) -coverprofile=coverage.txt -covermode atomic -``` - -### GitHub Actions Environment - -The CI environment uses: -- Consistent tool versions managed in `.versions.yaml` -- Shared build environment setup via `.github/actions/setup-build-env` -- Artifact uploads for test results and coverage reports -- Private repository access handled via SSH keys - -## 🐛 Debugging - -### Local Development Debugging - -1. **Tilt Debugging** - ```bash - # Start Tilt with Makefile (recommended) - make tilt-up - # Navigate to http://localhost:10350 - - # Or run Tilt in CI mode (no UI, good for debugging) - make tilt-ci - - # Stream logs for specific service - kubectl logs -f deployment/platform-connectors -n nvsentinel - - # Access Tilt logs and resource status - tilt get all - tilt logs platform-connectors - ``` - -2. **gRPC Debugging** - ```bash - # Use grpcurl to test endpoints - grpcurl -plaintext localhost:50051 list - grpcurl -plaintext localhost:50051 platformconnector.PlatformConnector/SendHealthEvent - ``` - -### Common Issues - -1. **Module Dependencies** - ```bash - # Clean module cache if dependency issues - go clean -modcache - go mod download - ``` - -2. **Private Repository Access** - ```bash - # Verify SSH key configuration - ssh -T git@github.com - - # Test access - git ls-remote git@github.com:dgxcloud/mk8s/k8s-addons/nvsentinel.git - ``` - -3. **Container Build Issues** - ```bash - # Clean Docker cache - docker system prune -f - - # Rebuild without cache - docker build --no-cache -t platform-connectors platform-connectors/ - ``` - -4. **Shellcheck Version Differences (Log Collector)** - ```bash - # GitHub Actions uses a specific shellcheck version from setup-build-env - # Local shellcheck version may differ, causing different linting results - - # Use standardized linting (matches GitHub Actions): - make -C log-collector lint-test # Standardized pattern - make log-collector-lint # Main Makefile delegation - - # Install shellcheck locally to match CI: - # macOS: brew install shellcheck - # Ubuntu: apt-get install shellcheck - # See: https://github.com/koalaman/shellcheck#installing - ``` - -### Debugging Setup Script Issues - -The `scripts/setup-dev-env.sh` script installs all development dependencies. If you encounter issues: - -#### Enable Debug Mode - -```bash -# Run with detailed debugging output -DEBUG=true make dev-env-setup AUTO_MODE=true - -# Or run the script directly -DEBUG=true AUTO_MODE=true ./scripts/setup-dev-env.sh -``` - -**Debug output includes**: -- Architecture detection (x86_64 → amd64, aarch64 → arm64) -- Architecture mappings for different tools (GO_ARCH vs PROTOC_ARCH) -- Complete URLs being constructed for downloads -- HTTP response codes from URL verification -- Version information from `.versions.yaml` - -#### Common Setup Issues - -**1. Download Failures (404 errors)** - -If you see errors like "HTTP 404 Not Found": - -```bash -# Enable debug mode to see the exact URL -DEBUG=true ./scripts/setup-dev-env.sh - -# Verify the URL manually -curl -I "https://github.com/koalaman/shellcheck/releases/download/v0.11.0/shellcheck-v0.11.0.linux.x86_64.tar.xz" - -# Check if the release exists on GitHub -# Visit: https://github.com///releases -``` - -**Common causes**: -- Version in `.versions.yaml` doesn't exist in GitHub releases -- Architecture-specific filename doesn't match release assets -- Tool project changed their release naming convention - -**2. Architecture Mismatch** - -Different tools use different architecture naming: -- **Go tools** (yq, kubectl, helm): `amd64`, `arm64`, `darwin` -- **Protocol Buffers**: `x86_64`, `aarch_64`, `osx-universal_binary` -- **Shellcheck**: `x86_64`, `aarch64`, `darwin` - -The script automatically maps these: -```bash -# See mappings in debug output -DEBUG=true ./scripts/setup-dev-env.sh -# Look for: "DEBUG: Architecture mappings: Raw ARCH: x86_64, GO_ARCH: amd64, PROTOC_ARCH: x86_64" -``` - -**3. Permission Issues** - -```bash -# If installation to /usr/local/bin fails -sudo make dev-env-setup AUTO_MODE=true - -# Or install to user directory (requires PATH modification) -mkdir -p ~/bin -export PATH="$HOME/bin:$PATH" -# Modify script to use ~/bin instead of /usr/local/bin -``` - -**4. Network/Proxy Issues** - -```bash -# Test connectivity to GitHub -curl -I https://github.com - -# If behind proxy, configure: -export HTTP_PROXY="http://proxy.example.com:8080" -export HTTPS_PROXY="http://proxy.example.com:8080" -export NO_PROXY="localhost,127.0.0.1" - -# Then retry -DEBUG=true make dev-env-setup AUTO_MODE=true -``` - -**5. Version Validation** - -```bash -# Check what versions are configured -cat .versions.yaml - -# Verify specific tool version exists -TOOL_VERSION=$(yq eval '.SHELLCHECK_VERSION' .versions.yaml) -echo "Checking shellcheck version: $TOOL_VERSION" -curl -I "https://github.com/koalaman/shellcheck/releases/tag/$TOOL_VERSION" - -# Update version if needed -yq eval '.SHELLCHECK_VERSION = "v0.11.0"' -i .versions.yaml -``` - -#### Testing URL Construction - -To test URL construction without running the full setup: - -```bash -# Source the script functions -source scripts/setup-dev-env.sh - -# Test specific tool URLs -echo "yq URL: $YQ_URL" -echo "kubectl URL: $KUBECTL_URL" -echo "protoc URL: $PROTOC_URL" -echo "shellcheck URL: $SHELLCHECK_URL" - -# Test URL accessibility -curl -I "$SHELLCHECK_URL" -``` - -#### Manual Verification Steps - -1. **Verify tool exists in releases**: - ```bash - # Check GitHub releases page - open "https://github.com/koalaman/shellcheck/releases" - - # Or use API - curl -s "https://api.github.com/repos/koalaman/shellcheck/releases/latest" | grep browser_download_url - ``` - -2. **Test download manually**: - ```bash - # Download specific asset - wget "https://github.com/koalaman/shellcheck/releases/download/v0.11.0/shellcheck-v0.11.0.linux.x86_64.tar.xz" - - # Verify archive integrity - tar -tzf shellcheck-v0.11.0.linux.x86_64.tar.xz - ``` - -3. **Validate script syntax**: - ```bash - # Check for syntax errors - bash -n scripts/setup-dev-env.sh - - # Run shellcheck on the script itself - shellcheck scripts/setup-dev-env.sh - ``` - -#### Reporting Setup Issues - -When reporting setup script issues, include: - -1. **Debug output**: - ```bash - DEBUG=true make dev-env-setup AUTO_MODE=true 2>&1 | tee setup-debug.log - ``` - -2. **Environment details**: - ```bash - echo "OS: $(uname -s)" - echo "Architecture: $(uname -m)" - echo "Shell: $SHELL" - echo "Bash version: $BASH_VERSION" - ``` - -3. **Version file content**: - ```bash - cat .versions.yaml - ``` - -4. **Failed URL** (from debug output): - ``` - Look for lines like: - ❌ ERROR: Failed to verify URL: https://... - HTTP Status: 404 Not Found - ``` - -This information helps diagnose whether the issue is: -- Version-specific (tool version doesn't exist) -- Architecture-specific (wrong filename for platform) -- Network-related (connectivity or proxy issues) -- Script bug (incorrect URL construction logic) - -## 🔧 Makefile Reference - -### Makefile Structure Overview - -The project uses a **unified Makefile structure** with shared patterns for consistency: - -#### Main Makefile (`./Makefile`) -Acts as the primary coordinator, delegating to specialized sub-Makefiles: - -```bash -make help # Show all available targets -make lint-test-all # Run full test suite (delegates to all modules) -make health-monitors-lint-test-all # Delegate to health-monitors/Makefile -make docker-all # Delegate to docker/Makefile -make dev-env # Delegate to dev/Makefile -make kubernetes-distro-lint # Delegate to distros/kubernetes/Makefile -``` - -#### Common Makefile Patterns (`common.mk`) -Shared build/test/Docker patterns for all Go modules: - -```bash -# Included by all Go modules with: include ../common.mk -# Provides consistent targets: -all # Default target: lint-test -lint-test # Full lint and test (matches CI) -vet, lint, test, coverage, build, binary # Individual steps -docker-build, docker-publish # Docker targets (if HAS_DOCKER=1) -setup-buildx, clean, help # Utility targets -``` - -#### Health Monitors Makefile (`health-monitors/Makefile`) -Coordinates all health monitoring modules: - -```bash -make -C health-monitors help # Show health monitor targets -make -C health-monitors lint-test-all # Test all health monitors -make -C health-monitors go-lint-test-all # Test Go health monitors -make -C health-monitors python-lint-test-all # Test Python health monitors -make -C health-monitors build-all # Build all health monitors -``` - -#### Individual Module Makefiles (Go modules use `common.mk`) -Each Go module includes `common.mk` for consistent patterns: - -```bash -# All Go modules have identical interface via common.mk -make -C health-monitors/syslog-health-monitor help # Help target -make -C health-monitors/syslog-health-monitor lint-test # Full lint-test -make -C platform-connectors lint-test # Same pattern -make -C health-events-analyzer lint-test # Same pattern - -# Individual development steps (common.mk patterns) -make -C health-monitors/syslog-health-monitor vet # go vet ./... -make -C health-monitors/syslog-health-monitor lint # golangci-lint run -make -C health-monitors/syslog-health-monitor test # gotestsum -make -C health-monitors/syslog-health-monitor coverage # coverage reports -make -C health-monitors/syslog-health-monitor build # go build ./... -make -C health-monitors/syslog-health-monitor binary # go build main binary -``` - -#### Docker Makefile (`docker/Makefile`) -**Delegation-based Docker build system:** - -```bash -make -C docker help # Show all Docker targets and configuration - -# Main build targets (delegates to individual modules) -make -C docker build-all # Build all images (delegates to modules) -make -C docker publish-all # Build and push all images -make -C docker setup-buildx # Setup Docker buildx builder - -# Group targets -make -C docker build-health-monitors # Build all health monitor images -make -C docker build-main-modules # Build all non-health-monitor images +Every source file (.go, .proto, .sh, Makefile) must include the Apache 2.0 license header. -# Individual module targets (delegates to module Makefiles) -make -C docker build-syslog-health-monitor # Calls module's docker-build -make -C docker build-csp-health-monitor # Calls module's docker-build -make -C docker build-gpu-health-monitor-dcgm3 # Calls module's docker-build-dcgm3 -make -C docker build-gpu-health-monitor-dcgm4 # Calls module's docker-build-dcgm4 -make -C docker build-platform-connectors # Calls module's docker-build -make -C docker build-health-events-analyzer # Calls module's docker-build -make -C docker build-log-collector # Calls module's docker-build +- **Go/Proto Template**: See `api/hack/boilerplate.go.txt`. +- **Year**: Ensure the copyright year is current. -# Publish targets (delegates to modules) -make -C docker publish-syslog-health-monitor # Calls module's docker-publish -make -C docker publish-all # Calls all modules' docker-publish - -# Utility targets -make -C docker clean # Remove all nvsentinel images -make -C docker list # List built nvsentinel images -``` - -**Key Features:** -- **Delegation-based**: Each module is single source of truth for its Docker config -- **Multi-platform builds**: `linux/arm64,linux/amd64` via `common.mk` -- **Build caching**: Registry-based cache for faster builds -- **Consistent patterns**: Go modules use `common.mk`, specialized for Python/shell -- **Dynamic tagging**: Uses `${SAFE_REF_NAME}` from branch/tag names -- **Registry integration**: Full NVCR.io paths and authentication - -#### Development Makefile (`dev/Makefile`) -Focused on development environment: - -```bash -make -C dev help # Show development targets -make -C dev env-up # Create cluster + start Tilt -make -C dev env-down # Stop Tilt + delete cluster -make -C dev cluster-create # Create Kind cluster -make -C dev tilt-up # Start Tilt -make -C dev cluster-status # Check cluster status -``` - -#### Kubernetes Makefile (`distros/kubernetes/Makefile`) -Helm and Kubernetes operations: - -```bash -make -C distros/kubernetes help # Show Kubernetes targets -make -C distros/kubernetes lint # Lint Helm charts -make -C distros/kubernetes helm-publish # Publish Helm chart -``` - -### Development Workflow Examples - -```bash -# 1. Full development cycle -make dev-env # Start development environment -make lint-test-all # Test all modules -make docker-all # Build containers (delegates to modules) -make dev-env-clean # Clean up - -# 2. Individual module development (common.mk patterns) -make platform-connectors-lint-test # Test specific Go module (main Makefile) -make -C platform-connectors lint-test # Test directly (common.mk pattern) -make -C platform-connectors docker-build # Build container (common.mk) - -# 3. Focused development on specific module (common.mk targets) -cd platform-connectors -make lint-test # Full module test -make vet # Quick syntax check -make test # Run tests only -make build # Build module -make binary # Build main binary - -# 4. Health monitors (coordination + common.mk patterns) -make health-monitors-lint-test-all # All health monitors -make -C health-monitors/syslog-health-monitor lint-test # Specific health monitor -``` - -### Build System Benefits - -All Go modules use consistent patterns via `common.mk`: -- **Consistent targets**: `lint-test`, `vet`, `lint`, `test`, `build`, `binary` -- **Docker integration**: `docker-build`, `docker-publish` (if `HAS_DOCKER=1`) -- **Unified configuration**: Same environment variables and build flags -- **Backwards compatibility**: Legacy targets (`image`, `publish`) still work - -## 📚 Common Tasks - -### Adding a New Dependency - -1. **For Go modules:** - ```bash - cd your-module/ - go get github.com/new/dependency@v1.2.3 - go mod tidy - ``` +--- -2. **For Python modules:** - ```bash - cd health-monitors/gpu-health-monitor/ - poetry add new-dependency - ``` +## Troubleshooting -### Updating Protobuf Definitions +### Tooling Not Found -1. **Edit `.proto` files in `protobufs/` directory** -2. **Regenerate code:** - ```bash - make protos-lint - ``` -3. **Update affected modules and test** +We use `.versions.yaml` to pin tool versions. Our Makefile attempts to use tools from your system path or download them to your Go bin directory. -### Adding New Configuration Options +- **Verify Installation**: `which protoc` or `which yq`. +- **Fix**: Ensure your `GOPATH/bin` is in your system `$PATH`: + ```bash + export PATH=$PATH:$(go env GOPATH)/bin + ``` -1. **Update Helm values in `distros/kubernetes/nvsentinel/values.yaml`** -2. **Update templates in `distros/kubernetes/nvsentinel/templates/`** -3. **Update module code to read new configuration** -4. **Test with Tilt or manual Helm install** +### Generated Code Out of Sync -### Performance Profiling +If the build fails or `make verify-codegen` returns an error, your generated artifacts are likely stale. ```bash -# Enable pprof in Go applications -import _ "net/http/pprof" +# Clean all generated files across the monorepo +make clean -# Access profiles -go tool pprof http://localhost:6060/debug/pprof/profile -go tool pprof http://localhost:6060/debug/pprof/heap +# Re-run the full pipeline +make code-gen ``` -### Database Schema Changes - -1. **Never break backward compatibility** -2. **Add fields with default values** -3. **Use MongoDB schema validation if needed** -4. **Test with existing data** - -## 🎯 Best Practices - -### Development Best Practices - -1. **Start Small**: Make incremental changes -2. **Test Early**: Write tests alongside code -3. **Document Changes**: Update relevant documentation -4. **Review Dependencies**: Minimize external dependencies -5. **Monitor Resources**: Be aware of CPU/memory usage - -### Kubernetes Best Practices - -1. **Resource Limits**: Always set resource requests/limits -2. **Health Checks**: Implement readiness and liveness probes -3. **Graceful Shutdown**: Handle SIGTERM properly -4. **Security Context**: Run with minimal privileges -5. **Observability**: Emit metrics and structured logs - -### MongoDB Best Practices - -1. **Indexes**: Create appropriate indexes for queries -2. **Connection Pooling**: Reuse connections efficiently -3. **Change Streams**: Use resume tokens for reliability -4. **Error Handling**: Handle network partitions gracefully - -**🎯 Usage Examples:** +### Dependency Issues -**Local Development Workflow:** -```bash -# Build for local testing (loads into local Docker daemon) -make -C docker build-syslog-health-monitor # Individual module -make -C docker build-all # All modules -make -C health-monitors/gpu-health-monitor docker-build-dcgm3 # Specific variant - -# Test the built images locally -# Test the built images locally -docker run ghcr.io/your-github-username/nvsentinel-syslog-health-monitor:local -``` +If you see "module not found" or checksum errors: -**CI/Production Workflow:** ```bash -# Environment setup (matches GitHub Actions) -export CONTAINER_REGISTRY="ghcr.io" -export CONTAINER_ORG="your-github-username" -export CI_COMMIT_REF_NAME="main" -# Authentication handled by docker login to ghcr.io - -# Build and push directly to registry (standardized patterns) -make -C docker publish-syslog-health-monitor # Individual module -make -C docker publish-all # All modules -make -C health-monitors/gpu-health-monitor docker-publish # Both DCGM variants +# Tidy all modules +make tidy ``` -**Development vs CI Behavior:** -```bash -# Development: Fast local build (recommended) -make -C health-monitors/syslog-health-monitor docker-build-local - -# Development: Full featured build (slower, like CI) -make -C health-monitors/syslog-health-monitor docker-build - -# CI/Production: Build and push with --push (standardized) -make -C health-monitors/syslog-health-monitor docker-publish -``` +--- -## 📞 Getting Help +## Getting Help -- **Internal Documentation**: Check module-specific READMEs and `make help` targets -- **GitHub Issues**: Report bugs and feature requests -- **Team Chat**: Reach out to the development team -- **Code Reviews**: Learn from feedback on pull requests -- **Makefile Help**: Use `make help` in any module for target documentation -- **Common Patterns**: All Go modules follow `common.mk` patterns for consistency +- **Issues**: [Create an issue](https://github.com/NVIDIA/device-api/issues/new) +- **Questions**: [Start a discussion](https://github.com/NVIDIA/device-api/discussions) +- **Security**: Please refer to [SECURITY](SECURITY.md) for reporting vulnerabilities. --- - -Happy coding! 🚀 - -For questions about this guide or the development process, please reach out to the NVSentinel development team. diff --git a/README.md b/README.md index 4ee29ecfe..dfd49ce46 100644 --- a/README.md +++ b/README.md @@ -1,246 +1,231 @@ # NVSentinel -[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) -[![Kubernetes](https://img.shields.io/badge/Kubernetes-1.25+-326CE5.svg?logo=kubernetes&logoColor=white)](https://kubernetes.io/) -[![Helm](https://img.shields.io/badge/Helm-3.0+-0F1689.svg?logo=helm&logoColor=white)](https://helm.sh/) +The NVIDIA Device API provides a Kubernetes-idiomatic Go SDK and Protobuf definitions for interacting with NVIDIA device resources. -**GPU Node Resilience System for Kubernetes** +**Node-local GPU device state management for Kubernetes** -NVSentinel is a comprehensive collection of Kubernetes services that automatically detect, classify, and remediate hardware and software faults in GPU nodes. Designed for GPU clusters, it ensures maximum uptime and seamless fault recovery in high-performance computing environments. +The NVIDIA Device API provides a standardized gRPC interface for observing and managing GPU device states in Kubernetes environments. It enables coordination between: -> [!WARNING] -> **Experimental Preview Release** -> This is an experimental/preview release of NVSentinel. Use at your own risk in production environments. The software is provided "as is" without warranties of any kind. Features, APIs, and configurations may change without notice in future releases. For production deployments, thoroughly test in non-critical environments first. +- **Providers** (health monitors like NVSentinel, DCGM) that detect GPU health issues +- **Consumers** (device plugins, DRA drivers) that need GPU health status for scheduling -## 🚀 Quick Start +## Overview -### Prerequisites - -- Kubernetes 1.25+ -- Helm 3.0+ -- NVIDIA GPU Operator (includes DCGM for GPU monitoring) +``` +┌─────────────────────────────────────────────────────────────┐ +│ GPU Node │ +│ │ +│ ┌─────────────────────────────────────────────────────────┐│ +│ │ Device API Server (DaemonSet) ││ +│ │ ││ +│ │ ┌─────────────────────────────────────────────────┐ ││ +│ │ │ GpuService (unified) │ ││ +│ │ │ Read: GetGpu, ListGpus, WatchGpus │ ││ +│ │ │ Write: CreateGpu, UpdateGpuStatus, DeleteGpu │ ││ +│ │ └────────────────────┬────────────────────────────┘ ││ +│ │ ▼ ││ +│ │ ┌──────────┐ ┌───────────────┐ ││ +│ │ │ Cache │◄───│ NVML Provider │ ││ +│ │ │ (RWLock) │ │ (optional) │ ││ +│ │ └──────────┘ └───────────────┘ ││ +│ └─────────────────────────────────────────────────────────┘│ +│ │ +│ Consumers: │ +│ ├── Device Plugins ────────► GetGpu, ListGpus, WatchGpus │ +│ └── DRA Drivers ───────────► GetGpu, ListGpus, WatchGpus │ +│ │ +│ Providers: │ +│ ├── NVSentinel (external) ─► CreateGpu, UpdateGpuStatus │ +│ ├── DCGM (external) ───────► CreateGpu, UpdateGpuStatus │ +│ └── NVML (built-in) ───────► GPU enumeration, XID monitor │ +└─────────────────────────────────────────────────────────────┘ +``` -### Installation +## Key Features -```bash -# Install from GitHub Container Registry -helm install nvsentinel oci://ghcr.io/nvidia/nvsentinel \ - --version v0.6.0 \ - --namespace nvsentinel \ - --create-namespace - -# View chart information -helm show chart oci://ghcr.io/nvidia/nvsentinel --version v0.6.0 -``` +- **Read-blocking semantics**: Consumer reads block during provider updates to prevent stale data +- **Multiple provider support**: Aggregate health status from NVSentinel, DCGM, or custom providers +- **Watch streams**: Real-time GPU state change notifications +- **Built-in NVML provider**: Optional GPU enumeration and XID monitoring without external providers +- **Prometheus metrics**: Full observability with alerting rules +- **Helm chart**: Production-ready Kubernetes deployment -## ✨ Key Features +## Repository Structure -- **🔍 Comprehensive Monitoring**: Real-time detection of GPU, NVSwitch, and system-level failures -- **🔧 Automated Remediation**: Intelligent fault handling with cordon, drain, and break-fix workflows -- **📦 Modular Architecture**: Pluggable health monitors with standardized gRPC interfaces -- **🔄 High Availability**: Kubernetes-native design with replica support and leader election -- **⚡ Real-time Processing**: Event-driven architecture with immediate fault response -- **📊 Persistent Storage**: MongoDB-based event store with change streams for real-time updates -- **🛡️ Graceful Handling**: Coordinated workload eviction with configurable timeouts -- **🏷️ Metadata Enrichment**: Automatic augmentation of health events with cloud provider and node metadata information +| Module | Description | +| :--- | :--- | +| [`api/`](./api) | Protobuf definitions and Go types for the Device API. | +| [`client-go/`](./client-go) | Kubernetes-style generated clients, informers, and listers. | +| [`code-generator/`](./code-generator) | Tools for generating NVIDIA-specific client logic. | +| [`cmd/device-api-server/`](./cmd/device-api-server) | Device API Server binary | +| [`pkg/deviceapiserver/`](./pkg/deviceapiserver) | Server implementation | +| [`charts/`](./charts) | Helm chart for Kubernetes deployment | -## 🧪 Complete Setup Guide +--- -For a full installation with all dependencies, follow these steps: +## Quick Start -### 1. Install cert-manager (for TLS) +### Deploy Device API Server ```bash -helm repo add jetstack https://charts.jetstack.io --force-update -helm upgrade --install cert-manager jetstack/cert-manager \ - --namespace cert-manager --create-namespace \ - --version v1.19.1 --set installCRDs=true \ - --wait +# Install with Helm +helm install device-api-server ./charts/device-api-server \ + --namespace device-api --create-namespace + +# Or with built-in NVML provider enabled +helm install device-api-server ./charts/device-api-server \ + --namespace device-api --create-namespace \ + --set nvml.enabled=true ``` -### 2. Install Prometheus (for metrics) +### Using the Go Client ```bash -helm repo add prometheus-community https://prometheus-community.github.io/helm-charts --force-update -helm upgrade --install prometheus prometheus-community/kube-prometheus-stack \ - --namespace monitoring --create-namespace \ - --set prometheus.enabled=true \ - --set alertmanager.enabled=false \ - --set grafana.enabled=false \ - --set kubeStateMetrics.enabled=false \ - --set nodeExporter.enabled=false \ - --wait +go get github.com/nvidia/device-api/api@latest ``` -### 3. Install NVSentinel - -```bash -NVSENTINEL_VERSION=v0.6.0 +```go +import ( + v1alpha1 "github.com/nvidia/device-api/api/gen/go/device/v1alpha1" +) +``` -helm upgrade --install nvsentinel oci://ghcr.io/nvidia/nvsentinel \ - --namespace nvsentinel --create-namespace \ - --version "$NVSENTINEL_VERSION" \ - --timeout 15m \ - --wait +### Example: List GPUs + +```go +package main + +import ( + "context" + "log" + + v1alpha1 "github.com/nvidia/device-api/api/gen/go/device/v1alpha1" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +func main() { + // Connect via Unix socket (recommended for node-local access) + conn, err := grpc.NewClient( + "unix:///var/run/device-api/device.sock", + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + log.Fatalf("failed to connect: %v", err) + } + defer conn.Close() + + client := v1alpha1.NewGpuServiceClient(conn) + + // List all GPUs + resp, err := client.ListGpus(context.Background(), &v1alpha1.ListGpusRequest{}) + if err != nil { + log.Fatalf("failed to list GPUs: %v", err) + } + + for _, gpu := range resp.GpuList.Items { + log.Printf("GPU: %s (UUID: %s)", gpu.Name, gpu.Spec.Uuid) + for _, cond := range gpu.Status.Conditions { + log.Printf(" %s: %s (%s)", cond.Type, cond.Status, cond.Reason) + } + } +} ``` -### 4. Verify Installation +### Using grpcurl ```bash -kubectl get pods -n nvsentinel -kubectl get nodes # Verify GPU nodes are visible +# List GPUs +grpcurl -plaintext localhost:50051 nvidia.device.v1alpha1.GpuService/ListGpus -# Run comprehensive validation -./scripts/validate-nvsentinel.sh --version v0.6.0 --verbose +# Watch for changes +grpcurl -plaintext localhost:50051 nvidia.device.v1alpha1.GpuService/WatchGpus ``` -> **Testing**: The example above uses default settings. For production, customize values for your environment. +## API Overview + +### GpuService + +The unified `GpuService` follows Kubernetes API conventions with standard CRUD methods: -> **Production**: By default, only health monitoring is enabled. Enable fault quarantine and remediation modules via Helm values. See [Configuration](#-configuration) below. +**Read Operations** (for consumers like device plugins and DRA drivers): -## 🎮 Try the Demo +| Method | Description | +|--------|-------------| +| `GetGpu` | Retrieves a single GPU resource by its unique name | +| `ListGpus` | Retrieves a list of all GPU resources | +| `WatchGpus` | Streams lifecycle events (ADDED, MODIFIED, DELETED) for GPU resources | -Want to see NVSentinel in action without GPU hardware? Try our **[Local Fault Injection Demo](demos/local-fault-injection-demo/README.md)**: +**Write Operations** (for providers like health monitors): + +| Method | Description | +|--------|-------------| +| `CreateGpu` | Register a new GPU with the server | +| `UpdateGpu` | Replace entire GPU resource | +| `UpdateGpuStatus` | Update GPU status only (acquires write lock) | +| `DeleteGpu` | Remove a GPU from the server | + +--- -- 🚀 **5-minute setup** - runs entirely in a local KIND cluster -- 🔍 **Real pipeline** - see fault detection → quarantine → node cordon -- 🎯 **No GPU required** - uses simulated DCGM for testing +## Development + +### Prerequisites + +- **Go**: `v1.25+` +- **Protoc**: Required for protobuf generation +- **golangci-lint**: Required for code quality checks +- **Make**: Used for orchestrating build and generation tasks +- **Helm 3.0+**: For chart development + +### Build ```bash -cd demos/local-fault-injection-demo -make demo # Automated: creates cluster, installs NVSentinel, injects fault, verifies cordon -``` +# Build everything +make build + +# Build server only +make build-server -Perfect for learning, presentations, or CI/CD testing! - -## 🏗️ Architecture - -NVSentinel follows a microservices architecture with modular health monitors and core processing modules: - -```mermaid -graph LR - subgraph "Health Monitors" - GPU["GPU Health Monitor
(DCGM Integration)"] - SYS["Syslog Health Monitor
(Journalctl)"] - CSP["CSP Health Monitor
(CSP APIs)"] - K8SOM["Kubernetes Object Monitor
(CEL Policies)"] - end - - subgraph "Core Processing" - PC["Platform Connectors
(gRPC Server)"] - STORE[("MongoDB Store
(Event Database)")] - FQ["Fault Quarantine
(Node Cordon)"] - ND["Node Drainer
(Workload Eviction)"] - FR["Fault Remediation
(Break-Fix Integration)"] - HEA["Health Events Analyzer
(Pattern Analysis)"] - LBL["Labeler
(Node Labels)"] - end - - subgraph "Kubernetes Cluster" - K8S["Kubernetes API
(Nodes, Pods, Events)"] - end - - GPU -->|gRPC| PC - SYS -->|gRPC| PC - CSP -->|gRPC| PC - K8SOM -->|gRPC| PC - - PC -->|persist| STORE - PC <-->|update status| K8S - - FQ -.->|watch changes| STORE - FQ -->|cordon| K8S - - ND -.->|watch changes| STORE - ND -->|drain| K8S - - FR -.->|watch changes| STORE - FR -->|create CRDs| K8S - - HEA -.->|watch changes| STORE - - LBL -->|update labels| K8S - - K8SOM -.->|watch changes| K8S +# Generate protobuf code +make code-gen ``` -**Data Flow**: -1. **Health Monitors** detect hardware/software faults and send events via gRPC to Platform Connectors -2. **Platform Connectors** validate, persist events to MongoDB, and update Kubernetes node conditions -3. **Core Modules** independently watch MongoDB change streams for relevant events -4. **Modules** interact with Kubernetes API to cordon, drain, label nodes, and create remediation CRDs -5. **Labeler** monitors pods to automatically label nodes with DCGM and driver versions - -> **Note**: All modules operate independently without direct communication. Coordination happens through MongoDB change streams and Kubernetes API. - -## ⚙️ Configuration - -NVSentinel is highly configurable with options for each module. For complete configuration documentation, see the **[Helm Chart README](distros/kubernetes/README.md)**. - -### Quick Configuration Overview - -```yaml -global: - dryRun: false # Test mode - log actions without executing - - # Health Monitors (enabled by default) - gpuHealthMonitor: - enabled: true - syslogHealthMonitor: - enabled: true - - # Core Modules (disabled by default - enable for production) - faultQuarantine: - enabled: false - nodeDrainer: - enabled: false - faultRemediation: - enabled: false - janitor: - enabled: false - mongodbStore: - enabled: false +### Test + +```bash +# Run all tests +make test + +# Run server tests only +make test-server ``` -**Configuration Resources**: -- **[Helm Chart Configuration Guide](distros/kubernetes/README.md#configuration)**: Complete configuration reference -- **[values-full.yaml](distros/kubernetes/nvsentinel/values-full.yaml)**: Detailed reference with all options -- **[values.yaml](distros/kubernetes/nvsentinel/values.yaml)**: Default values +### Lint -## 📦 Module Details +```bash +make lint +``` -For detailed module configuration, see the **[Helm Chart Configuration Guide](distros/kubernetes/README.md#module-specific-configuration)**. +--- -### 🔍 Health Monitors +## Documentation -- **[GPU Health Monitor](docs/gpu-health-monitor.md)**: Monitors GPU hardware health via DCGM - detects thermal issues, ECC errors, and XID events -- **[Syslog Health Monitor](docs/syslog-health-monitor.md)**: Analyzes system logs for hardware and software fault patterns via journalctl -- **CSP Health Monitor**: Integrates with cloud provider APIs (GCP/AWS) for maintenance events -- **[Kubernetes Object Monitor](docs/kubernetes-object-monitor.md)**: Policy-based monitoring for any Kubernetes resource using CEL expressions +- **[API Reference](docs/api/device-api-server.md)** - Complete gRPC API documentation +- **[Operations Guide](docs/operations/device-api-server.md)** - Deployment, configuration, monitoring +- **[Helm Chart](charts/device-api-server/README.md)** - Chart configuration reference +- **[Design Documents](docs/design/)** - Architecture and design decisions -### 🏗️ Core Modules +The `client-go` module includes several examples for how to use the generated clients: -- **[Platform Connectors](docs/platform-connectors.md)**: Receives health events from monitors via gRPC, persists to MongoDB, and updates Kubernetes node status -- **[Fault Quarantine](docs/fault-quarantine.md)**: Watches MongoDB for health events and cordons nodes based on configurable CEL rules -- **[Node Drainer](docs/node-drainer.md)**: Gracefully evicts workloads from cordoned nodes with per-namespace eviction strategies -- **[Fault Remediation](docs/fault-remediation.md)**: Triggers external break-fix systems by creating maintenance CRDs after drain completion -- **Janitor**: Executes node reboots and terminations via cloud provider APIs -- **Health Events Analyzer**: Analyzes event patterns and generates recommended actions -- **[Event Exporter](docs/event-exporter.md)**: Streams health events to external systems in CloudEvents format -- **MongoDB Store**: Persistent storage for health events with real-time change streams -- **[Labeler](docs/labeler.md)**: Automatically labels nodes with DCGM and driver versions for self-configuration -- **[Metadata Collector](docs/metadata-collector.md)**: Gathers GPU and NVSwitch topology information -- **[Log Collection](docs/log-collection.md)**: Collects diagnostic logs and GPU reports for troubleshooting +* **Standard Client**: Basic CRUD operations. +* **Shared Informers**: High-performance caching for controllers. +* **Watch**: Real-time event streaming via gRPC. -## 📋 Requirements +See the [examples](./client-go/examples) directory for details. -- **Kubernetes**: 1.25 or later -- **Helm**: 3.0 or later -- **NVIDIA GPU Operator**: For GPU monitoring capabilities (includes DCGM) -- **Storage**: Persistent storage for MongoDB (recommended 10GB+) -- **Network**: Cluster networking for inter-service communication +--- -## 🤝 Contributing +## Contributing We welcome contributions! Here's how to get started: @@ -271,7 +256,9 @@ All contributors must sign their commits (DCO). See the contributing guide for d - 👀 **Watch** for updates on releases and announcements - 🔗 **Share** NVSentinel with others who might benefit -## 📄 License +--- + +## License This project is licensed under the Apache License 2.0 - see the [LICENSE](LICENSE) file for details. diff --git a/api/DEVELOPMENT.md b/api/DEVELOPMENT.md new file mode 100644 index 000000000..6514333e9 --- /dev/null +++ b/api/DEVELOPMENT.md @@ -0,0 +1,43 @@ +# NVIDIA Device API: API Development + +--- + +## Workflow + +Follow these steps to add new resources or update existing fields: + +1. **Proto Definitions**: Update `proto/device/${VERSION}/${TYPE}.proto`. Ensure Protobuf field numbers are never reused or changed once released. +2. **Go Definitions**: Update `device/${VERSION}/${TYPE}_types.go`. Ensure you include the necessary marker comments (e.g., `// +k8s:deepcopy-gen`) required by the generators. +3. **Registration**: If adding a new **Kind**, register it in `device/${VERSION}/register.go` within the `addKnownTypes` function. +4. **Conversion Logic**: Update the mapping interface in `device/${VERSION}/converter.go`. See [Goverter](https://github.com/jmattheis/goverter) documentation for additional details. +5. **Generate**: Run `make code-gen`. This orchestrates `protoc`, `deepcopy-gen`, and `goverter` to refresh all artifacts. + +--- + +## Conventions + +### Kubernetes Resource Model (KRM) + +- **Spec vs Status**: Use `Spec` for intended state and `Status` for observed state. Please refer to the [Kubernetes Resource Model](https://github.com/kubernetes/design-proposals-archive/blob/main/architecture/resource-management.md) for additional details. +- **Conditions**: Use the standard `metav1.Condition` slice in the Status block. + +### Protocol Buffer Guidelines + +When modifying protocol buffer definitions, adhere to the following: + +- **Naming**: Use `snake_case` for field names and `CamelCase` for messages/services. +- **Documentation**: Every message and field must have a comment describing its purpose. +- **Style**: Follow the [Google Protocol Buffer Style Guide](https://protobuf.dev/programming-guides/style/) + +--- + +## Housekeeping + +If your generated files are out of sync or contain stale data, you can purge the local generated artifacts: + +```bash +# Removes generated code (bindings, deepcopy, goverter) +make clean +``` + +> [!TIP] For broader repository issues or environment-wise troubleshooting, please refer to the [Root Development Guide](../DEVELOPMENT.md). diff --git a/api/Makefile b/api/Makefile index 4bd7652ef..00e70de3f 100644 --- a/api/Makefile +++ b/api/Makefile @@ -16,41 +16,25 @@ # Configuration # ============================================================================== -# Directories -PROTO_DIR := proto -GEN_DIR := gen/go +MODULE_NAME = $(shell basename $(shell pwd)) # Shell setup SHELL = /usr/bin/env bash -o pipefail .SHELLFLAGS = -ec -# Location to install dependencies to -LOCALBIN ?= $(shell pwd)/bin -$(LOCALBIN): - mkdir -p $(LOCALBIN) - -# Tool Binaries -PROTOC_GEN_GO ?= $(LOCALBIN)/protoc-gen-go -PROTOC_GEN_GO_GRPC ?= $(LOCALBIN)/protoc-gen-go-grpc - -# Tool Versions (load from ../.versions.yaml) -# Requires yq to be installed: brew install yq (macOS) or see https://github.com/mikefarah/yq -YQ := $(shell command -v yq 2> /dev/null) -ifndef YQ -$(error yq is required but not found. Install it with: brew install yq (macOS) or see https://github.com/mikefarah/yq) -endif +# Tool versions +CONTROLLER_TOOLS_VERSION ?= v0.17.3 -VERSIONS_FILE := ../.versions.yaml - -PROTOC_GEN_GO_VERSION := $(shell $(YQ) '.protobuf.protoc_gen_go' $(VERSIONS_FILE)) -PROTOC_GEN_GO_GRPC_VERSION := $(shell $(YQ) '.protobuf.protoc_gen_go_grpc' $(VERSIONS_FILE)) +# Tool binaries +LOCALBIN ?= $(shell pwd)/bin +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen # ============================================================================== # Targets # ============================================================================== .PHONY: all -all: protos-generate build +all: code-gen manifests test build ## Run code generation, compile all code, and execute tests. ##@ General @@ -60,66 +44,50 @@ help: ## Display this help. ##@ Development +.PHONY: code-gen +code-gen: ## Run all code generation targets (deepcopy, protobuf). + @echo "Generating code ($(MODULE_NAME))..." + ./hack/update-codegen.sh + @echo "Synchronizing module dependencies..." + go mod tidy + +.PHONY: manifests +manifests: controller-gen ## Generate CRD manifests using controller-gen. + @echo "Generating CRD manifests..." + $(CONTROLLER_GEN) crd paths="./nvsentinel/..." output:crd:artifacts:config=../deployments/helm/nvsentinel/crds + +.PHONY: generate +generate: code-gen manifests ## Run all code generation (deepcopy + CRD manifests). + .PHONY: build -build: +build: ## Compile all Go code after generation to verify type safety. + @echo "Compiling ($(MODULE_NAME))..." go build -v ./... -.PHONY: protos-generate -protos-generate: $(PROTOC_GEN_GO) $(PROTOC_GEN_GO_GRPC) protos-clean ## Generate Go code from Proto definitions. - @echo "Generating Proto code..." - @mkdir -p $(GEN_DIR) - cd proto && \ - protoc \ - -I . \ - -I ../$(THIRD_PARTY_DIR) \ - --plugin="protoc-gen-go=$(PROTOC_GEN_GO)" \ - --plugin="protoc-gen-go-grpc=$(PROTOC_GEN_GO_GRPC)" \ - --go_out=../$(GEN_DIR) \ - --go_opt=paths=source_relative \ - --go-grpc_out=../$(GEN_DIR) \ - --go-grpc_opt=paths=source_relative \ - device/v1alpha1/*.proto && \ - protoc \ - -I . \ - -I ../$(THIRD_PARTY_DIR) \ - --plugin="protoc-gen-go=$(PROTOC_GEN_GO)" \ - --plugin="protoc-gen-go-grpc=$(PROTOC_GEN_GO_GRPC)" \ - --go_out=../$(GEN_DIR) \ - --go_opt=paths=source_relative \ - --go-grpc_out=../$(GEN_DIR) \ - --go-grpc_opt=paths=source_relative \ - csp/v1alpha1/*.proto - @echo "Cleaning up dependencies..." - go mod tidy - @echo "Done." - -.PHONY: protos-clean -protos-clean: ## Remove generated code. - @echo "Cleaning old generated Proto code..." - rm -rf $(GEN_DIR) - @echo "Done." - -##@ Build Dependencies - -$(PROTOC_GEN_GO): - $(call go-install-tool,$(PROTOC_GEN_GO),google.golang.org/protobuf/cmd/protoc-gen-go,$(PROTOC_GEN_GO_VERSION)) - -$(PROTOC_GEN_GO_GRPC): - $(call go-install-tool,$(PROTOC_GEN_GO_GRPC),google.golang.org/grpc/cmd/protoc-gen-go-grpc,$(PROTOC_GEN_GO_GRPC_VERSION)) - -# go-install-tool macro -# $1 - target path with name of binary -# $2 - package url which can be installed -# $3 - specific version of package -define go-install-tool -@[ -f "$(1)-$(3)" ] || { \ -set -e; \ -mkdir -p $(LOCALBIN); \ -package=$(2)@$(3) ;\ -echo "Downloading $${package}" ;\ -rm -f $(1) || true ;\ -GOBIN=$(LOCALBIN) go install $${package} ;\ -mv $(1) $(1)-$(3) ;\ -} ;\ -ln -sf $(1)-$(3) $(1) -endef +.PHONY: test +test: ## Run unit tests and generate coverage after code generation. + @echo "Testing ($(MODULE_NAME))..." + go test -v $$(go list ./... | grep -v /gen/) -coverprofile cover.out + +.PHONY: lint +lint: ## Run golangci-lint. + @echo "Linting ($(MODULE_NAME))..." + golangci-lint run ./... + +.PHONY: clean +clean: ## Remove all generated code. + @echo "Cleaning generated code ($(MODULE_NAME))..." + rm -rf gen/go + find . -name "zz_generated.*.go" -delete + +##@ Tools + +.PHONY: controller-gen +controller-gen: $(CONTROLLER_GEN) ## Install controller-gen locally. + +$(CONTROLLER_GEN): $(LOCALBIN) + @echo "Installing controller-gen $(CONTROLLER_TOOLS_VERSION)..." + GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) + +$(LOCALBIN): + mkdir -p $(LOCALBIN) diff --git a/api/README.md b/api/README.md new file mode 100644 index 000000000..747c2690d --- /dev/null +++ b/api/README.md @@ -0,0 +1,54 @@ + + +# NVIDIA Device API: API Definitions + +The `api` module contains the canonical API definitions for the **NVIDIA Device API**. It serves as the single source of truth for resource schemas, gRPC wire formats, and Kubernetes-native Go types. + +--- + +## Structure + +* **`device/`**: Contains the **Kubernetes Resource Model (KRM)** definitions. These types implement the `runtime.Object` interface. +* **`proto/`**: Contains the **Language-Agnostic Definitions**. Protobuf messages and gRPC service definitions that define the node-local communication contract. +* **`gen/go/`**: Contains the **Bindings**. The output of the `protoc` compiler (`.pb.go` and `_grpc.pb.go`). + +--- + +## Code Generation + +This module relies on three distinct generation phases to maintain type safety: + +1. **Protobuf**: Compiles `.proto` files into Go structs. +2. **DeepCopy**: Generates `zz_generated.deepcopy.go` to support Kubernetes object manipulation. +3. **Conversion**: Generates `zz_generated.goverter.go` to map between Protobuf messages and KRM Go types (using Goverter). + +> [!NOTE] +> While the pipeline is automated, the **mapping** between Protobufs and Kubernetes types is manually defined in `api/device/${VERSION}/converter.go`. This file serves as the configuration for Goverter, allowing you to define custom transformation rules for fields. + +To run the full pipeline: + +```bash +make code-gen +``` + +--- + +## Development + +Refer to this module's [Development Guide](DEVELOPMENT.md) for instructions on adding new fields, defining resources, or updating the conversion mapping. + +--- diff --git a/api/device/v1alpha1/converter.go b/api/device/v1alpha1/converter.go new file mode 100644 index 000000000..61e99b5b0 --- /dev/null +++ b/api/device/v1alpha1/converter.go @@ -0,0 +1,139 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + pb "github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1" + "google.golang.org/protobuf/types/known/timestamppb" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Converter is the interface used to generate type conversion methods between the +// Kubernetes Resource Model structs and the Protobuf message structs. +// +// goverter:converter +// goverter:output:file ./zz_generated.goverter.go +// goverter:extend FromProtobufTypeMeta FromProtobufListTypeMeta FromProtobufTimestamp ToProtobufTimestamp +// goverter:useZeroValueOnPointerInconsistency +type Converter interface { + // FromProtobuf converts a protobuf Gpu message into a GPU object. + // + // goverter:map . TypeMeta | FromProtobufTypeMeta + // goverter:map Metadata ObjectMeta + FromProtobuf(source *pb.Gpu) GPU + + // ToProtobuf converts a GPU object into a protobuf Gpu message. + // + // goverter:map ObjectMeta Metadata + // goverter:ignore state sizeCache unknownFields + ToProtobuf(source GPU) *pb.Gpu + + // FromProtobufList converts a protobuf GpuList message into a GPUList object. + // + // goverter:map . TypeMeta | FromProtobufListTypeMeta + // goverter:map Metadata ListMeta + FromProtobufList(source *pb.GpuList) *GPUList + + // ToProtobufList converts a GPUList object into a protobuf GpuList message. + // + // goverter:map ListMeta Metadata + // goverter:ignore state sizeCache unknownFields + ToProtobufList(source *GPUList) *pb.GpuList + + // FromProtobufObjectMeta converts a protobuf ObjectMeta into a metav1.ObjectMeta object. + // + // goverter:ignore GenerateName UID Generation CreationTimestamp DeletionTimestamp DeletionGracePeriodSeconds + // goverter:ignore Labels Annotations OwnerReferences Finalizers ManagedFields SelfLink + FromProtobufObjectMeta(source *pb.ObjectMeta) metav1.ObjectMeta + + // ToProtobufObjectMeta converts a metav1.ObjectMeta into a protobuf Object message. + // + // goverter:ignore state sizeCache unknownFields + ToProtobufObjectMeta(source metav1.ObjectMeta) *pb.ObjectMeta + + // FromProtobufListMeta converts a protobuf ListMeta into a metav1.ListMeta object. + // + // goverter:ignore SelfLink Continue RemainingItemCount + FromProtobufListMeta(source *pb.ListMeta) metav1.ListMeta + + // ToProtobufListMeta converts a metav1.ListMeta into a protobuf ListMeta message. + // + // goverter:ignore state sizeCache unknownFields + ToProtobufListMeta(source metav1.ListMeta) *pb.ListMeta + + // FromProtobufSpec converts a protobuf GpuSpec message into a GPUSpec object. + // + // goverter:map Uuid UUID + FromProtobufSpec(source *pb.GpuSpec) GPUSpec + + // ToProtobufSpec converts a GPUSpec object into a protobuf GpuSpec message. + // + // goverter:map UUID Uuid + // goverter:ignore state sizeCache unknownFields + ToProtobufSpec(source GPUSpec) *pb.GpuSpec + + // FromProtobufStatus converts a protobuf GpuStatus message into a GPUStatus object. + FromProtobufStatus(source *pb.GpuStatus) GPUStatus + + // ToProtobufStatus converts a GPUStatus object into a protobuf GpuStatus message. + // + // goverter:ignore state sizeCache unknownFields + ToProtobufStatus(source GPUStatus) *pb.GpuStatus + + // FromProtobufCondition converts a protobuf Condition message into a metav1.Condition object. + // + // goverter:ignore ObservedGeneration + // Note: ObservedGeneration is specific to k8s and not found in the protobuf Condition message. + FromProtobufCondition(source *pb.Condition) metav1.Condition + + // ToProtobufCondition converts a metav1.Condition object into a protobuf Condition message. + // + // goverter:ignore state sizeCache unknownFields + ToProtobufCondition(source metav1.Condition) *pb.Condition +} + +// FromProtobufTypeMeta generates the standard TypeMeta for the root GPU resource. +func FromProtobufTypeMeta(_ *pb.Gpu) metav1.TypeMeta { + return metav1.TypeMeta{ + Kind: "GPU", + APIVersion: SchemeGroupVersion.String(), + } +} + +// FromProtobufListTypeMeta generates the standard TypeMeta for the GPUList resource. +func FromProtobufListTypeMeta(_ *pb.GpuList) metav1.TypeMeta { + return metav1.TypeMeta{ + Kind: "GPUList", + APIVersion: SchemeGroupVersion.String(), + } +} + +// FromProtobufTimestamp converts a protobuf Timestamp message to a metav1.Time. +func FromProtobufTimestamp(source *timestamppb.Timestamp) metav1.Time { + if source == nil { + return metav1.Time{} + } + + return metav1.NewTime(source.AsTime()) +} + +// ToProtobufTimestamp converts a metav1.Time to a protobuf Timestamp message. +func ToProtobufTimestamp(source metav1.Time) *timestamppb.Timestamp { + if source.IsZero() { + return nil + } + + return timestamppb.New(source.Time) +} diff --git a/api/device/v1alpha1/doc.go b/api/device/v1alpha1/doc.go new file mode 100644 index 000000000..15af71d19 --- /dev/null +++ b/api/device/v1alpha1/doc.go @@ -0,0 +1,19 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +groupName=device.nvidia.com + +// Package v1alpha1 contains the API Schema definitions for the device.nvidia.com v1alpha1 API group. +// This package includes types for GPU resources and conversions between Kubernetes and protobuf representations. +package v1alpha1 diff --git a/api/device/v1alpha1/gpu_conversion.go b/api/device/v1alpha1/gpu_conversion.go new file mode 100644 index 000000000..582b80ff2 --- /dev/null +++ b/api/device/v1alpha1/gpu_conversion.go @@ -0,0 +1,60 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !goverter + +package v1alpha1 + +import pb "github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1" + +// converter is the singleton instance of the generated Converter implementation. +var converter Converter = &ConverterImpl{} + +// FromProto converts a protobuf Gpu message pointer to a GPU object pointer. +func FromProto(in *pb.Gpu) *GPU { + if in == nil { + return nil + } + + val := converter.FromProtobuf(in) + + return &val +} + +// ToProto converts a GPU object pointer to a protobuf Gpu message pointer. +func ToProto(in *GPU) *pb.Gpu { + if in == nil { + return nil + } + + return converter.ToProtobuf(*in) +} + +// FromProtoList converts a protobuf GpuList message pointer to a GPUList object pointer. +func FromProtoList(in *pb.GpuList) *GPUList { + if in == nil { + return nil + } + + return converter.FromProtobufList(in) +} + +// ToProtoList converts a GPUList object pointer to a protobuf GpuList message pointer. +func ToProtoList(in *GPUList) *pb.GpuList { + if in == nil { + return nil + } + + return converter.ToProtobufList(in) +} diff --git a/api/device/v1alpha1/gpu_conversion_test.go b/api/device/v1alpha1/gpu_conversion_test.go new file mode 100644 index 000000000..00767b5ea --- /dev/null +++ b/api/device/v1alpha1/gpu_conversion_test.go @@ -0,0 +1,153 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + pb "github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/timestamppb" +) + +var lastTransitionTime = time.Date(2025, 1, 1, 12, 0, 0, 0, time.UTC) + +func TestGPUConversion_Nil(t *testing.T) { + if out := FromProto(nil); out != nil { + t.Errorf("FromProto(nil): got %#v, expected nil", out) + } + if out := ToProto(nil); out != nil { + t.Errorf("ToProto(nil): got %#v, expected nil", out) + } + if out := FromProtoList(nil); out != nil { + t.Errorf("FromProtoList(nil): got %#v, expected nil", out) + } + if out := ToProtoList(nil); out != nil { + t.Errorf("ToProtoList(nil): got %#v, expected nil", out) + } +} + +func TestGPUConversion(t *testing.T) { + + protoIn := &pb.Gpu{ + Metadata: &pb.ObjectMeta{ + Name: "gpu-1111", + ResourceVersion: "1", + }, + Spec: &pb.GpuSpec{ + Uuid: "GPU-1111", + }, + Status: &pb.GpuStatus{ + Conditions: []*pb.Condition{ + { + Type: "Ready", + Status: "False", + LastTransitionTime: timestamppb.New(lastTransitionTime), + Reason: "DriverCrash", + Message: "The driver has stopped responding.", + }, + }, + RecommendedAction: "ResetGPU", + }, + } + + goStruct := FromProto(protoIn) + + expectedName := strings.ToLower(protoIn.Metadata.Name) + if goStruct.ObjectMeta.Name != expectedName { + t.Errorf("ObjectMeta.Name conversion failed: got %q, want %q", + goStruct.ObjectMeta.Name, expectedName) + } + + expectedResourceVersion := "1" + if goStruct.ObjectMeta.ResourceVersion != expectedResourceVersion { + t.Errorf("ObjectMeta.ResourceVersion conversion failed: got %q, want %q", + goStruct.ObjectMeta.ResourceVersion, expectedResourceVersion) + } + + protoOut := ToProto(goStruct) + + if diff := cmp.Diff(protoIn, protoOut, protocmp.Transform()); diff != "" { + t.Errorf("Conversion failed (-want +got):\n%s", diff) + } +} + +func TestGPUListConversion(t *testing.T) { + protoIn := &pb.GpuList{ + Metadata: &pb.ListMeta{ + ResourceVersion: "2", + }, + Items: []*pb.Gpu{ + { + Metadata: &pb.ObjectMeta{ + Name: "gpu-1111", + ResourceVersion: "1", + }, + Spec: &pb.GpuSpec{ + Uuid: "GPU-1111", + }, + Status: &pb.GpuStatus{ + Conditions: []*pb.Condition{ + { + Type: "Ready", + Status: "True", + LastTransitionTime: timestamppb.New(lastTransitionTime), + Reason: "DriverReady", + Message: "Driver is posting ready status.", + }, + }, + }, + }, + { + Metadata: &pb.ObjectMeta{ + Name: "gpu-2222", + ResourceVersion: "2", + }, + Spec: &pb.GpuSpec{ + Uuid: "GPU-2222", + }, + Status: &pb.GpuStatus{ + Conditions: []*pb.Condition{ + { + Type: "HardwareFailure", + Status: "True", + LastTransitionTime: timestamppb.New(lastTransitionTime.Add(1 * time.Minute)), + Reason: "DoubleBitECCError", + Message: "Double Bit ECC error detected.", + }, + }, + RecommendedAction: "RebootNode", + }, + }, + }, + } + + goList := FromProtoList(protoIn) + + expectedResourceVersion := "2" + if goList.ListMeta.ResourceVersion != expectedResourceVersion { + t.Errorf("ListMeta.ResourceVersion conversion failed: got %q, want %q", + goList.ListMeta.ResourceVersion, expectedResourceVersion) + } + + protoOut := ToProtoList(goList) + + if diff := cmp.Diff(protoIn, protoOut, protocmp.Transform()); diff != "" { + t.Errorf("Conversion failed (-want +got):\n%s", diff) + } +} diff --git a/api/device/v1alpha1/gpu_types.go b/api/device/v1alpha1/gpu_types.go new file mode 100644 index 000000000..ad17601e0 --- /dev/null +++ b/api/device/v1alpha1/gpu_types.go @@ -0,0 +1,71 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// GPUSpec defines the desired state of the GPU. +// +// +k8s:deepcopy-gen=true +type GPUSpec struct { + // UUID is the physical hardware UUID of the GPU. + // + // Format: 'GPU-' + // (e.g., 'GPU-a1b2c3d4-e5f6-a7b8-c9d0-e1f2a3b4c5d6'). + UUID string `json:"uuid"` +} + +// GPUStatus defines the observed state of the GPU. +// +// +k8s:deepcopy-gen=true +type GPUStatus struct { + // Conditions is an array of current gpu conditions. + // + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` + + // RecommendedAction is a suggestion given the current state. + // + // +optional + RecommendedAction string `json:"recommendedAction,omitempty"` +} + +// GPU represents a single GPU resource. +// +// +genclient +// +genclient:nonNamespaced +// +genclient:onlyVerbs=get,list,watch +// +genclient:noStatus +// +k8s:deepcopy-gen=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type GPU struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec GPUSpec `json:"spec,omitempty"` + Status GPUStatus `json:"status,omitempty"` +} + +// GPUList contains a list of GPU resources. +// +// +k8s:deepcopy-gen=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type GPUList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []GPU `json:"items"` +} diff --git a/api/device/v1alpha1/register.go b/api/device/v1alpha1/register.go new file mode 100644 index 000000000..c6f6ea098 --- /dev/null +++ b/api/device/v1alpha1/register.go @@ -0,0 +1,46 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var SchemeGroupVersion = schema.GroupVersion{Group: "device.nvidia.com", Version: "v1alpha1"} + +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} + +var ( + // SchemeBuilder initializes a scheme builder. + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + + // AddToScheme adds the type in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) + +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &GPU{}, + &GPUList{}, + ) + + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + + return nil +} diff --git a/api/device/v1alpha1/zz_generated.deepcopy.go b/api/device/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000..0c399eb3e --- /dev/null +++ b/api/device/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,125 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GPU) DeepCopyInto(out *GPU) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPU. +func (in *GPU) DeepCopy() *GPU { + if in == nil { + return nil + } + out := new(GPU) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GPU) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GPUList) DeepCopyInto(out *GPUList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]GPU, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUList. +func (in *GPUList) DeepCopy() *GPUList { + if in == nil { + return nil + } + out := new(GPUList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GPUList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GPUSpec) DeepCopyInto(out *GPUSpec) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUSpec. +func (in *GPUSpec) DeepCopy() *GPUSpec { + if in == nil { + return nil + } + out := new(GPUSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GPUStatus) DeepCopyInto(out *GPUStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUStatus. +func (in *GPUStatus) DeepCopy() *GPUStatus { + if in == nil { + return nil + } + out := new(GPUStatus) + in.DeepCopyInto(out) + return out +} diff --git a/api/device/v1alpha1/zz_generated.goverter.go b/api/device/v1alpha1/zz_generated.goverter.go new file mode 100644 index 000000000..72c8b2cb6 --- /dev/null +++ b/api/device/v1alpha1/zz_generated.goverter.go @@ -0,0 +1,144 @@ +// Code generated by github.com/jmattheis/goverter, DO NOT EDIT. +//go:build !goverter + +package v1alpha1 + +import ( + v1alpha1 "github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type ConverterImpl struct{} + +func (c *ConverterImpl) FromProtobuf(source *v1alpha1.Gpu) GPU { + var v1alpha1GPU GPU + if source != nil { + v1alpha1GPU.TypeMeta = FromProtobufTypeMeta(source) + v1alpha1GPU.ObjectMeta = c.FromProtobufObjectMeta((*source).Metadata) + v1alpha1GPU.Spec = c.FromProtobufSpec((*source).Spec) + v1alpha1GPU.Status = c.FromProtobufStatus((*source).Status) + } + return v1alpha1GPU +} +func (c *ConverterImpl) FromProtobufCondition(source *v1alpha1.Condition) v1.Condition { + var v1Condition v1.Condition + if source != nil { + v1Condition.Type = (*source).Type + v1Condition.Status = v1.ConditionStatus((*source).Status) + v1Condition.LastTransitionTime = FromProtobufTimestamp((*source).LastTransitionTime) + v1Condition.Reason = (*source).Reason + v1Condition.Message = (*source).Message + } + return v1Condition +} +func (c *ConverterImpl) FromProtobufList(source *v1alpha1.GpuList) *GPUList { + var pV1alpha1GPUList *GPUList + if source != nil { + var v1alpha1GPUList GPUList + v1alpha1GPUList.TypeMeta = FromProtobufListTypeMeta(source) + v1alpha1GPUList.ListMeta = c.FromProtobufListMeta((*source).Metadata) + if (*source).Items != nil { + v1alpha1GPUList.Items = make([]GPU, len((*source).Items)) + for i := 0; i < len((*source).Items); i++ { + v1alpha1GPUList.Items[i] = c.FromProtobuf((*source).Items[i]) + } + } + pV1alpha1GPUList = &v1alpha1GPUList + } + return pV1alpha1GPUList +} +func (c *ConverterImpl) FromProtobufListMeta(source *v1alpha1.ListMeta) v1.ListMeta { + var v1ListMeta v1.ListMeta + if source != nil { + v1ListMeta.ResourceVersion = (*source).ResourceVersion + } + return v1ListMeta +} +func (c *ConverterImpl) FromProtobufObjectMeta(source *v1alpha1.ObjectMeta) v1.ObjectMeta { + var v1ObjectMeta v1.ObjectMeta + if source != nil { + v1ObjectMeta.Name = (*source).Name + v1ObjectMeta.Namespace = (*source).Namespace + v1ObjectMeta.ResourceVersion = (*source).ResourceVersion + } + return v1ObjectMeta +} +func (c *ConverterImpl) FromProtobufSpec(source *v1alpha1.GpuSpec) GPUSpec { + var v1alpha1GPUSpec GPUSpec + if source != nil { + v1alpha1GPUSpec.UUID = (*source).Uuid + } + return v1alpha1GPUSpec +} +func (c *ConverterImpl) FromProtobufStatus(source *v1alpha1.GpuStatus) GPUStatus { + var v1alpha1GPUStatus GPUStatus + if source != nil { + if (*source).Conditions != nil { + v1alpha1GPUStatus.Conditions = make([]v1.Condition, len((*source).Conditions)) + for i := 0; i < len((*source).Conditions); i++ { + v1alpha1GPUStatus.Conditions[i] = c.FromProtobufCondition((*source).Conditions[i]) + } + } + v1alpha1GPUStatus.RecommendedAction = (*source).RecommendedAction + } + return v1alpha1GPUStatus +} +func (c *ConverterImpl) ToProtobuf(source GPU) *v1alpha1.Gpu { + var v1alpha1Gpu v1alpha1.Gpu + v1alpha1Gpu.Metadata = c.ToProtobufObjectMeta(source.ObjectMeta) + v1alpha1Gpu.Spec = c.ToProtobufSpec(source.Spec) + v1alpha1Gpu.Status = c.ToProtobufStatus(source.Status) + return &v1alpha1Gpu +} +func (c *ConverterImpl) ToProtobufCondition(source v1.Condition) *v1alpha1.Condition { + var v1alpha1Condition v1alpha1.Condition + v1alpha1Condition.Type = source.Type + v1alpha1Condition.Status = string(source.Status) + v1alpha1Condition.LastTransitionTime = ToProtobufTimestamp(source.LastTransitionTime) + v1alpha1Condition.Reason = source.Reason + v1alpha1Condition.Message = source.Message + return &v1alpha1Condition +} +func (c *ConverterImpl) ToProtobufList(source *GPUList) *v1alpha1.GpuList { + var pV1alpha1GpuList *v1alpha1.GpuList + if source != nil { + var v1alpha1GpuList v1alpha1.GpuList + v1alpha1GpuList.Metadata = c.ToProtobufListMeta((*source).ListMeta) + if (*source).Items != nil { + v1alpha1GpuList.Items = make([]*v1alpha1.Gpu, len((*source).Items)) + for i := 0; i < len((*source).Items); i++ { + v1alpha1GpuList.Items[i] = c.ToProtobuf((*source).Items[i]) + } + } + pV1alpha1GpuList = &v1alpha1GpuList + } + return pV1alpha1GpuList +} +func (c *ConverterImpl) ToProtobufListMeta(source v1.ListMeta) *v1alpha1.ListMeta { + var v1alpha1ListMeta v1alpha1.ListMeta + v1alpha1ListMeta.ResourceVersion = source.ResourceVersion + return &v1alpha1ListMeta +} +func (c *ConverterImpl) ToProtobufObjectMeta(source v1.ObjectMeta) *v1alpha1.ObjectMeta { + var v1alpha1ObjectMeta v1alpha1.ObjectMeta + v1alpha1ObjectMeta.Name = source.Name + v1alpha1ObjectMeta.ResourceVersion = source.ResourceVersion + v1alpha1ObjectMeta.Namespace = source.Namespace + return &v1alpha1ObjectMeta +} +func (c *ConverterImpl) ToProtobufSpec(source GPUSpec) *v1alpha1.GpuSpec { + var v1alpha1GpuSpec v1alpha1.GpuSpec + v1alpha1GpuSpec.Uuid = source.UUID + return &v1alpha1GpuSpec +} +func (c *ConverterImpl) ToProtobufStatus(source GPUStatus) *v1alpha1.GpuStatus { + var v1alpha1GpuStatus v1alpha1.GpuStatus + if source.Conditions != nil { + v1alpha1GpuStatus.Conditions = make([]*v1alpha1.Condition, len(source.Conditions)) + for i := 0; i < len(source.Conditions); i++ { + v1alpha1GpuStatus.Conditions[i] = c.ToProtobufCondition(source.Conditions[i]) + } + } + v1alpha1GpuStatus.RecommendedAction = source.RecommendedAction + return &v1alpha1GpuStatus +} diff --git a/api/gen/go/device/v1alpha1/gpu.pb.go b/api/gen/go/device/v1alpha1/gpu.pb.go index 35e35a5e8..896c3df44 100644 --- a/api/gen/go/device/v1alpha1/gpu.pb.go +++ b/api/gen/go/device/v1alpha1/gpu.pb.go @@ -23,6 +23,7 @@ package v1alpha1 import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + emptypb "google.golang.org/protobuf/types/known/emptypb" timestamppb "google.golang.org/protobuf/types/known/timestamppb" reflect "reflect" sync "sync" @@ -36,17 +37,261 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +// ObjectMeta is a subset of k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. +type ObjectMeta struct { + state protoimpl.MessageState `protogen:"open.v1"` + // name is the unique logical identifier of the resource. Must be unique within a namespace. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // resource_version represents the internal version of this object. + // + // Value must be treated as opaque by clients and passed unmodified back to the server. + // Populated by the system. + // Read-only. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + ResourceVersion string `protobuf:"bytes,2,opt,name=resource_version,json=resourceVersion,proto3" json:"resource_version,omitempty"` + // namespace defines the space within which each name must be unique. An empty namespace is + // equivalent to the "default" namespace, but "default" is the canonical representation. + // Not all objects are required to be scoped to a namespace - the value of this field for + // those objects will be empty. + Namespace string `protobuf:"bytes,3,opt,name=namespace,proto3" json:"namespace,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ObjectMeta) Reset() { + *x = ObjectMeta{} + mi := &file_device_v1alpha1_gpu_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ObjectMeta) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ObjectMeta) ProtoMessage() {} + +func (x *ObjectMeta) ProtoReflect() protoreflect.Message { + mi := &file_device_v1alpha1_gpu_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ObjectMeta.ProtoReflect.Descriptor instead. +func (*ObjectMeta) Descriptor() ([]byte, []int) { + return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{0} +} + +func (x *ObjectMeta) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *ObjectMeta) GetResourceVersion() string { + if x != nil { + return x.ResourceVersion + } + return "" +} + +func (x *ObjectMeta) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + +// ListMeta is a subset of k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta. +type ListMeta struct { + state protoimpl.MessageState `protogen:"open.v1"` + // resource_version identifies the version of the list snapshot. + // Clients can use this version to establish a watch from a consistent point in time. + // + // Value must be treated as opaque by clients and passed unmodified back to the server. + // Populated by the system. + // Read-only. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + ResourceVersion string `protobuf:"bytes,1,opt,name=resource_version,json=resourceVersion,proto3" json:"resource_version,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListMeta) Reset() { + *x = ListMeta{} + mi := &file_device_v1alpha1_gpu_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListMeta) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListMeta) ProtoMessage() {} + +func (x *ListMeta) ProtoReflect() protoreflect.Message { + mi := &file_device_v1alpha1_gpu_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListMeta.ProtoReflect.Descriptor instead. +func (*ListMeta) Descriptor() ([]byte, []int) { + return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{1} +} + +func (x *ListMeta) GetResourceVersion() string { + if x != nil { + return x.ResourceVersion + } + return "" +} + +// GetOptions is the standard query options to the standard gRPC get call. +type GetOptions struct { + state protoimpl.MessageState `protogen:"open.v1"` + // resource_version sets a constraint on what resource versions a request may be served from. + // + // Optional. Defaults to unset. + ResourceVersion string `protobuf:"bytes,1,opt,name=resource_version,json=resourceVersion,proto3" json:"resource_version,omitempty"` + // namespace sets a constraint on what namespaces a request may be served from. + // An empty namespace is treated as a request for all namespaces. + // + // Optional. Defaults to unset. + Namespace string `protobuf:"bytes,2,opt,name=namespace,proto3" json:"namespace,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetOptions) Reset() { + *x = GetOptions{} + mi := &file_device_v1alpha1_gpu_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetOptions) ProtoMessage() {} + +func (x *GetOptions) ProtoReflect() protoreflect.Message { + mi := &file_device_v1alpha1_gpu_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetOptions.ProtoReflect.Descriptor instead. +func (*GetOptions) Descriptor() ([]byte, []int) { + return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{2} +} + +func (x *GetOptions) GetResourceVersion() string { + if x != nil { + return x.ResourceVersion + } + return "" +} + +func (x *GetOptions) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + +// ListOptions is the query options to a standard gRPC list call. It is a subset of k8s.io/apimachinery/pkg/apis/meta/v1.ListOptions. +type ListOptions struct { + state protoimpl.MessageState `protogen:"open.v1"` + // resource_version sets a constraint on what resource versions a request may be served from. + // + // Optional. Defaults to unset. + ResourceVersion string `protobuf:"bytes,1,opt,name=resource_version,json=resourceVersion,proto3" json:"resource_version,omitempty"` + // namespace sets a constraint on what namespaces a request may be served from. + // An empty namespace is treated as a request for all namespaces. + // + // Optional. Defaults to unset. + Namespace string `protobuf:"bytes,2,opt,name=namespace,proto3" json:"namespace,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListOptions) Reset() { + *x = ListOptions{} + mi := &file_device_v1alpha1_gpu_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListOptions) ProtoMessage() {} + +func (x *ListOptions) ProtoReflect() protoreflect.Message { + mi := &file_device_v1alpha1_gpu_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListOptions.ProtoReflect.Descriptor instead. +func (*ListOptions) Descriptor() ([]byte, []int) { + return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{3} +} + +func (x *ListOptions) GetResourceVersion() string { + if x != nil { + return x.ResourceVersion + } + return "" +} + +func (x *ListOptions) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + // Gpu represents a single GPU resource. // // Its structure follows the Kubernetes Resource Model pattern (Spec/Status). +// +// The resource name (metadata.name) is typically the lowercased GPU UUID, +// but may take other forms. +// +// Example: "gpu-a1b2c3d4-e5f6-a7b8-c9d0-e1f2a3b4c5d6" type Gpu struct { - state protoimpl.MessageState `protogen:"open.v1"` - // name is the unique logical identifier of the GPU resource. - // - // This is typically the lowercased GPU UUID, but may take other forms. - // - // Example: "gpu-a1b2c3d4-e5f6-a7b8-c9d0-e1f2a3b4c5d6" - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Metadata *ObjectMeta `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` // spec defines the identity and desired attributes of the GPU resource. Spec *GpuSpec `protobuf:"bytes,2,opt,name=spec,proto3" json:"spec,omitempty"` // status contains the most recently observed state of the GPU resource. @@ -58,7 +303,7 @@ type Gpu struct { func (x *Gpu) Reset() { *x = Gpu{} - mi := &file_device_v1alpha1_gpu_proto_msgTypes[0] + mi := &file_device_v1alpha1_gpu_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -70,7 +315,7 @@ func (x *Gpu) String() string { func (*Gpu) ProtoMessage() {} func (x *Gpu) ProtoReflect() protoreflect.Message { - mi := &file_device_v1alpha1_gpu_proto_msgTypes[0] + mi := &file_device_v1alpha1_gpu_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -83,14 +328,14 @@ func (x *Gpu) ProtoReflect() protoreflect.Message { // Deprecated: Use Gpu.ProtoReflect.Descriptor instead. func (*Gpu) Descriptor() ([]byte, []int) { - return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{0} + return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{4} } -func (x *Gpu) GetName() string { +func (x *Gpu) GetMetadata() *ObjectMeta { if x != nil { - return x.Name + return x.Metadata } - return "" + return nil } func (x *Gpu) GetSpec() *GpuSpec { @@ -109,16 +354,17 @@ func (x *Gpu) GetStatus() *GpuStatus { // GpuList is a collection of GPU resources. type GpuList struct { - state protoimpl.MessageState `protogen:"open.v1"` + state protoimpl.MessageState `protogen:"open.v1"` + Metadata *ListMeta `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` // items is the list of GPU resources. - Items []*Gpu `protobuf:"bytes,1,rep,name=items,proto3" json:"items,omitempty"` + Items []*Gpu `protobuf:"bytes,2,rep,name=items,proto3" json:"items,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *GpuList) Reset() { *x = GpuList{} - mi := &file_device_v1alpha1_gpu_proto_msgTypes[1] + mi := &file_device_v1alpha1_gpu_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -130,7 +376,7 @@ func (x *GpuList) String() string { func (*GpuList) ProtoMessage() {} func (x *GpuList) ProtoReflect() protoreflect.Message { - mi := &file_device_v1alpha1_gpu_proto_msgTypes[1] + mi := &file_device_v1alpha1_gpu_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -143,7 +389,14 @@ func (x *GpuList) ProtoReflect() protoreflect.Message { // Deprecated: Use GpuList.ProtoReflect.Descriptor instead. func (*GpuList) Descriptor() ([]byte, []int) { - return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{1} + return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{5} +} + +func (x *GpuList) GetMetadata() *ListMeta { + if x != nil { + return x.Metadata + } + return nil } func (x *GpuList) GetItems() []*Gpu { @@ -166,7 +419,7 @@ type GpuSpec struct { func (x *GpuSpec) Reset() { *x = GpuSpec{} - mi := &file_device_v1alpha1_gpu_proto_msgTypes[2] + mi := &file_device_v1alpha1_gpu_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -178,7 +431,7 @@ func (x *GpuSpec) String() string { func (*GpuSpec) ProtoMessage() {} func (x *GpuSpec) ProtoReflect() protoreflect.Message { - mi := &file_device_v1alpha1_gpu_proto_msgTypes[2] + mi := &file_device_v1alpha1_gpu_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -191,7 +444,7 @@ func (x *GpuSpec) ProtoReflect() protoreflect.Message { // Deprecated: Use GpuSpec.ProtoReflect.Descriptor instead. func (*GpuSpec) Descriptor() ([]byte, []int) { - return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{2} + return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{6} } func (x *GpuSpec) GetUuid() string { @@ -214,7 +467,7 @@ type GpuStatus struct { func (x *GpuStatus) Reset() { *x = GpuStatus{} - mi := &file_device_v1alpha1_gpu_proto_msgTypes[3] + mi := &file_device_v1alpha1_gpu_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -226,7 +479,7 @@ func (x *GpuStatus) String() string { func (*GpuStatus) ProtoMessage() {} func (x *GpuStatus) ProtoReflect() protoreflect.Message { - mi := &file_device_v1alpha1_gpu_proto_msgTypes[3] + mi := &file_device_v1alpha1_gpu_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -239,7 +492,7 @@ func (x *GpuStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use GpuStatus.ProtoReflect.Descriptor instead. func (*GpuStatus) Descriptor() ([]byte, []int) { - return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{3} + return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{7} } func (x *GpuStatus) GetConditions() []*Condition { @@ -279,7 +532,7 @@ type Condition struct { func (x *Condition) Reset() { *x = Condition{} - mi := &file_device_v1alpha1_gpu_proto_msgTypes[4] + mi := &file_device_v1alpha1_gpu_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -291,7 +544,7 @@ func (x *Condition) String() string { func (*Condition) ProtoMessage() {} func (x *Condition) ProtoReflect() protoreflect.Message { - mi := &file_device_v1alpha1_gpu_proto_msgTypes[4] + mi := &file_device_v1alpha1_gpu_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -304,7 +557,7 @@ func (x *Condition) ProtoReflect() protoreflect.Message { // Deprecated: Use Condition.ProtoReflect.Descriptor instead. func (*Condition) Descriptor() ([]byte, []int) { - return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{4} + return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{8} } func (x *Condition) GetType() string { @@ -346,14 +599,16 @@ func (x *Condition) GetMessage() string { type GetGpuRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // The unique resource name of the GPU to retrieve. - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // opts contains the standard query and scoping options for the request. + Opts *GetOptions `protobuf:"bytes,2,opt,name=opts,proto3" json:"opts,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *GetGpuRequest) Reset() { *x = GetGpuRequest{} - mi := &file_device_v1alpha1_gpu_proto_msgTypes[5] + mi := &file_device_v1alpha1_gpu_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -365,7 +620,7 @@ func (x *GetGpuRequest) String() string { func (*GetGpuRequest) ProtoMessage() {} func (x *GetGpuRequest) ProtoReflect() protoreflect.Message { - mi := &file_device_v1alpha1_gpu_proto_msgTypes[5] + mi := &file_device_v1alpha1_gpu_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -378,7 +633,7 @@ func (x *GetGpuRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetGpuRequest.ProtoReflect.Descriptor instead. func (*GetGpuRequest) Descriptor() ([]byte, []int) { - return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{5} + return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{9} } func (x *GetGpuRequest) GetName() string { @@ -388,6 +643,13 @@ func (x *GetGpuRequest) GetName() string { return "" } +func (x *GetGpuRequest) GetOpts() *GetOptions { + if x != nil { + return x.Opts + } + return nil +} + // GetGpuResponse contains the requested GPU resource. type GetGpuResponse struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -399,7 +661,7 @@ type GetGpuResponse struct { func (x *GetGpuResponse) Reset() { *x = GetGpuResponse{} - mi := &file_device_v1alpha1_gpu_proto_msgTypes[6] + mi := &file_device_v1alpha1_gpu_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -411,7 +673,7 @@ func (x *GetGpuResponse) String() string { func (*GetGpuResponse) ProtoMessage() {} func (x *GetGpuResponse) ProtoReflect() protoreflect.Message { - mi := &file_device_v1alpha1_gpu_proto_msgTypes[6] + mi := &file_device_v1alpha1_gpu_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -424,7 +686,7 @@ func (x *GetGpuResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetGpuResponse.ProtoReflect.Descriptor instead. func (*GetGpuResponse) Descriptor() ([]byte, []int) { - return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{6} + return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{10} } func (x *GetGpuResponse) GetGpu() *Gpu { @@ -435,18 +697,17 @@ func (x *GetGpuResponse) GetGpu() *Gpu { } // ListGpusRequest specifies the criteria for listing GPU resources. -// -// NOTE: The request is currently empty, but reserved for future support -// of filtering and pagination. type ListGpusRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` + state protoimpl.MessageState `protogen:"open.v1"` + // opts contains the standard query and scoping options for the list. + Opts *ListOptions `protobuf:"bytes,1,opt,name=opts,proto3" json:"opts,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *ListGpusRequest) Reset() { *x = ListGpusRequest{} - mi := &file_device_v1alpha1_gpu_proto_msgTypes[7] + mi := &file_device_v1alpha1_gpu_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -458,7 +719,7 @@ func (x *ListGpusRequest) String() string { func (*ListGpusRequest) ProtoMessage() {} func (x *ListGpusRequest) ProtoReflect() protoreflect.Message { - mi := &file_device_v1alpha1_gpu_proto_msgTypes[7] + mi := &file_device_v1alpha1_gpu_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -471,7 +732,14 @@ func (x *ListGpusRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListGpusRequest.ProtoReflect.Descriptor instead. func (*ListGpusRequest) Descriptor() ([]byte, []int) { - return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{7} + return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{11} +} + +func (x *ListGpusRequest) GetOpts() *ListOptions { + if x != nil { + return x.Opts + } + return nil } // ListGpusResponse contains the list of GPU resources. @@ -485,7 +753,7 @@ type ListGpusResponse struct { func (x *ListGpusResponse) Reset() { *x = ListGpusResponse{} - mi := &file_device_v1alpha1_gpu_proto_msgTypes[8] + mi := &file_device_v1alpha1_gpu_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -497,7 +765,7 @@ func (x *ListGpusResponse) String() string { func (*ListGpusResponse) ProtoMessage() {} func (x *ListGpusResponse) ProtoReflect() protoreflect.Message { - mi := &file_device_v1alpha1_gpu_proto_msgTypes[8] + mi := &file_device_v1alpha1_gpu_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -510,7 +778,7 @@ func (x *ListGpusResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListGpusResponse.ProtoReflect.Descriptor instead. func (*ListGpusResponse) Descriptor() ([]byte, []int) { - return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{8} + return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{12} } func (x *ListGpusResponse) GetGpuList() *GpuList { @@ -521,18 +789,17 @@ func (x *ListGpusResponse) GetGpuList() *GpuList { } // WatchGpusRequest specifies the parameters for the watch stream. -// -// NOTE: The request is currently empty, but reserved for future support -// of filtering and resource versioning (resumption). type WatchGpusRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` + state protoimpl.MessageState `protogen:"open.v1"` + // opts contains the standard query and scoping options for the watch. + Opts *ListOptions `protobuf:"bytes,1,opt,name=opts,proto3" json:"opts,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *WatchGpusRequest) Reset() { *x = WatchGpusRequest{} - mi := &file_device_v1alpha1_gpu_proto_msgTypes[9] + mi := &file_device_v1alpha1_gpu_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -544,7 +811,7 @@ func (x *WatchGpusRequest) String() string { func (*WatchGpusRequest) ProtoMessage() {} func (x *WatchGpusRequest) ProtoReflect() protoreflect.Message { - mi := &file_device_v1alpha1_gpu_proto_msgTypes[9] + mi := &file_device_v1alpha1_gpu_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -557,7 +824,14 @@ func (x *WatchGpusRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use WatchGpusRequest.ProtoReflect.Descriptor instead. func (*WatchGpusRequest) Descriptor() ([]byte, []int) { - return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{9} + return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{13} +} + +func (x *WatchGpusRequest) GetOpts() *ListOptions { + if x != nil { + return x.Opts + } + return nil } // WatchGpusResponse describes a change event for a GPU resource. @@ -582,7 +856,7 @@ type WatchGpusResponse struct { func (x *WatchGpusResponse) Reset() { *x = WatchGpusResponse{} - mi := &file_device_v1alpha1_gpu_proto_msgTypes[10] + mi := &file_device_v1alpha1_gpu_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -594,7 +868,7 @@ func (x *WatchGpusResponse) String() string { func (*WatchGpusResponse) ProtoMessage() {} func (x *WatchGpusResponse) ProtoReflect() protoreflect.Message { - mi := &file_device_v1alpha1_gpu_proto_msgTypes[10] + mi := &file_device_v1alpha1_gpu_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -607,7 +881,7 @@ func (x *WatchGpusResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use WatchGpusResponse.ProtoReflect.Descriptor instead. func (*WatchGpusResponse) Descriptor() ([]byte, []int) { - return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{10} + return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{14} } func (x *WatchGpusResponse) GetType() string { @@ -624,22 +898,398 @@ func (x *WatchGpusResponse) GetObject() *Gpu { return nil } +// CreateGpuRequest contains the GPU to create. +type CreateGpuRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The GPU to create. + // Required: metadata.name, spec.uuid + Gpu *Gpu `protobuf:"bytes,1,opt,name=gpu,proto3" json:"gpu,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateGpuRequest) Reset() { + *x = CreateGpuRequest{} + mi := &file_device_v1alpha1_gpu_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateGpuRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateGpuRequest) ProtoMessage() {} + +func (x *CreateGpuRequest) ProtoReflect() protoreflect.Message { + mi := &file_device_v1alpha1_gpu_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateGpuRequest.ProtoReflect.Descriptor instead. +func (*CreateGpuRequest) Descriptor() ([]byte, []int) { + return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{15} +} + +func (x *CreateGpuRequest) GetGpu() *Gpu { + if x != nil { + return x.Gpu + } + return nil +} + +// CreateGpuResponse contains the created GPU. +type CreateGpuResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // gpu is the created GPU with server-assigned fields (resource_version). + Gpu *Gpu `protobuf:"bytes,1,opt,name=gpu,proto3" json:"gpu,omitempty"` + // created is true if a new GPU was created, false if it already existed. + // + // When false, the returned gpu contains the existing GPU data. + // This enables idempotent registration patterns. + Created bool `protobuf:"varint,2,opt,name=created,proto3" json:"created,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateGpuResponse) Reset() { + *x = CreateGpuResponse{} + mi := &file_device_v1alpha1_gpu_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateGpuResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateGpuResponse) ProtoMessage() {} + +func (x *CreateGpuResponse) ProtoReflect() protoreflect.Message { + mi := &file_device_v1alpha1_gpu_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateGpuResponse.ProtoReflect.Descriptor instead. +func (*CreateGpuResponse) Descriptor() ([]byte, []int) { + return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{16} +} + +func (x *CreateGpuResponse) GetGpu() *Gpu { + if x != nil { + return x.Gpu + } + return nil +} + +func (x *CreateGpuResponse) GetCreated() bool { + if x != nil { + return x.Created + } + return false +} + +// UpdateGpuRequest contains the GPU to update. +type UpdateGpuRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The GPU to update. + // Required: metadata.name + // Optional: resource_version for optimistic concurrency + Gpu *Gpu `protobuf:"bytes,1,opt,name=gpu,proto3" json:"gpu,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *UpdateGpuRequest) Reset() { + *x = UpdateGpuRequest{} + mi := &file_device_v1alpha1_gpu_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *UpdateGpuRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UpdateGpuRequest) ProtoMessage() {} + +func (x *UpdateGpuRequest) ProtoReflect() protoreflect.Message { + mi := &file_device_v1alpha1_gpu_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UpdateGpuRequest.ProtoReflect.Descriptor instead. +func (*UpdateGpuRequest) Descriptor() ([]byte, []int) { + return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{17} +} + +func (x *UpdateGpuRequest) GetGpu() *Gpu { + if x != nil { + return x.Gpu + } + return nil +} + +// UpdateGpuResponse contains the updated GPU. +type UpdateGpuResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // gpu is the updated GPU. + Gpu *Gpu `protobuf:"bytes,1,opt,name=gpu,proto3" json:"gpu,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *UpdateGpuResponse) Reset() { + *x = UpdateGpuResponse{} + mi := &file_device_v1alpha1_gpu_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *UpdateGpuResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UpdateGpuResponse) ProtoMessage() {} + +func (x *UpdateGpuResponse) ProtoReflect() protoreflect.Message { + mi := &file_device_v1alpha1_gpu_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UpdateGpuResponse.ProtoReflect.Descriptor instead. +func (*UpdateGpuResponse) Descriptor() ([]byte, []int) { + return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{18} +} + +func (x *UpdateGpuResponse) GetGpu() *Gpu { + if x != nil { + return x.Gpu + } + return nil +} + +// UpdateGpuStatusRequest contains the status update for a GPU. +type UpdateGpuStatusRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The name of the GPU to update. + // Required. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // The new status to set. + // This completely replaces the existing status. + // Required. + Status *GpuStatus `protobuf:"bytes,2,opt,name=status,proto3" json:"status,omitempty"` + // Expected resource_version for optimistic concurrency. + // If set and doesn't match current version, returns ABORTED. + // Optional. + ResourceVersion string `protobuf:"bytes,3,opt,name=resource_version,json=resourceVersion,proto3" json:"resource_version,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *UpdateGpuStatusRequest) Reset() { + *x = UpdateGpuStatusRequest{} + mi := &file_device_v1alpha1_gpu_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *UpdateGpuStatusRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UpdateGpuStatusRequest) ProtoMessage() {} + +func (x *UpdateGpuStatusRequest) ProtoReflect() protoreflect.Message { + mi := &file_device_v1alpha1_gpu_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UpdateGpuStatusRequest.ProtoReflect.Descriptor instead. +func (*UpdateGpuStatusRequest) Descriptor() ([]byte, []int) { + return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{19} +} + +func (x *UpdateGpuStatusRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *UpdateGpuStatusRequest) GetStatus() *GpuStatus { + if x != nil { + return x.Status + } + return nil +} + +func (x *UpdateGpuStatusRequest) GetResourceVersion() string { + if x != nil { + return x.ResourceVersion + } + return "" +} + +// UpdateGpuStatusResponse contains the updated GPU. +type UpdateGpuStatusResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // gpu is the updated GPU. + Gpu *Gpu `protobuf:"bytes,1,opt,name=gpu,proto3" json:"gpu,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *UpdateGpuStatusResponse) Reset() { + *x = UpdateGpuStatusResponse{} + mi := &file_device_v1alpha1_gpu_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *UpdateGpuStatusResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UpdateGpuStatusResponse) ProtoMessage() {} + +func (x *UpdateGpuStatusResponse) ProtoReflect() protoreflect.Message { + mi := &file_device_v1alpha1_gpu_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UpdateGpuStatusResponse.ProtoReflect.Descriptor instead. +func (*UpdateGpuStatusResponse) Descriptor() ([]byte, []int) { + return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{20} +} + +func (x *UpdateGpuStatusResponse) GetGpu() *Gpu { + if x != nil { + return x.Gpu + } + return nil +} + +// DeleteGpuRequest specifies which GPU to delete. +type DeleteGpuRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The unique name of the GPU to delete. + // Required. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteGpuRequest) Reset() { + *x = DeleteGpuRequest{} + mi := &file_device_v1alpha1_gpu_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteGpuRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteGpuRequest) ProtoMessage() {} + +func (x *DeleteGpuRequest) ProtoReflect() protoreflect.Message { + mi := &file_device_v1alpha1_gpu_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteGpuRequest.ProtoReflect.Descriptor instead. +func (*DeleteGpuRequest) Descriptor() ([]byte, []int) { + return file_device_v1alpha1_gpu_proto_rawDescGZIP(), []int{21} +} + +func (x *DeleteGpuRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + var File_device_v1alpha1_gpu_proto protoreflect.FileDescriptor const file_device_v1alpha1_gpu_proto_rawDesc = "" + "\n" + - "\x19device/v1alpha1/gpu.proto\x12\x1anvidia.nvsentinel.v1alpha1\x1a\x1fgoogle/protobuf/timestamp.proto\"\x91\x01\n" + - "\x03Gpu\x12\x12\n" + - "\x04name\x18\x01 \x01(\tR\x04name\x127\n" + - "\x04spec\x18\x02 \x01(\v2#.nvidia.nvsentinel.v1alpha1.GpuSpecR\x04spec\x12=\n" + - "\x06status\x18\x03 \x01(\v2%.nvidia.nvsentinel.v1alpha1.GpuStatusR\x06status\"@\n" + - "\aGpuList\x125\n" + - "\x05items\x18\x01 \x03(\v2\x1f.nvidia.nvsentinel.v1alpha1.GpuR\x05items\"\x1d\n" + + "\x19device/v1alpha1/gpu.proto\x12\x16nvidia.device.v1alpha1\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"i\n" + + "\n" + + "ObjectMeta\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12)\n" + + "\x10resource_version\x18\x02 \x01(\tR\x0fresourceVersion\x12\x1c\n" + + "\tnamespace\x18\x03 \x01(\tR\tnamespace\"5\n" + + "\bListMeta\x12)\n" + + "\x10resource_version\x18\x01 \x01(\tR\x0fresourceVersion\"U\n" + + "\n" + + "GetOptions\x12)\n" + + "\x10resource_version\x18\x01 \x01(\tR\x0fresourceVersion\x12\x1c\n" + + "\tnamespace\x18\x02 \x01(\tR\tnamespace\"V\n" + + "\vListOptions\x12)\n" + + "\x10resource_version\x18\x01 \x01(\tR\x0fresourceVersion\x12\x1c\n" + + "\tnamespace\x18\x02 \x01(\tR\tnamespace\"\xb5\x01\n" + + "\x03Gpu\x12>\n" + + "\bmetadata\x18\x01 \x01(\v2\".nvidia.device.v1alpha1.ObjectMetaR\bmetadata\x123\n" + + "\x04spec\x18\x02 \x01(\v2\x1f.nvidia.device.v1alpha1.GpuSpecR\x04spec\x129\n" + + "\x06status\x18\x03 \x01(\v2!.nvidia.device.v1alpha1.GpuStatusR\x06status\"z\n" + + "\aGpuList\x12<\n" + + "\bmetadata\x18\x01 \x01(\v2 .nvidia.device.v1alpha1.ListMetaR\bmetadata\x121\n" + + "\x05items\x18\x02 \x03(\v2\x1b.nvidia.device.v1alpha1.GpuR\x05items\"\x1d\n" + "\aGpuSpec\x12\x12\n" + - "\x04uuid\x18\x01 \x01(\tR\x04uuid\"\x81\x01\n" + - "\tGpuStatus\x12E\n" + + "\x04uuid\x18\x01 \x01(\tR\x04uuid\"}\n" + + "\tGpuStatus\x12A\n" + "\n" + - "conditions\x18\x01 \x03(\v2%.nvidia.nvsentinel.v1alpha1.ConditionR\n" + + "conditions\x18\x01 \x03(\v2!.nvidia.device.v1alpha1.ConditionR\n" + "conditions\x12-\n" + "\x12recommended_action\x18\x02 \x01(\tR\x11recommendedAction\"\xb7\x01\n" + "\tCondition\x12\x12\n" + @@ -647,23 +1297,47 @@ const file_device_v1alpha1_gpu_proto_rawDesc = "" + "\x06status\x18\x02 \x01(\tR\x06status\x12L\n" + "\x14last_transition_time\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\x12lastTransitionTime\x12\x16\n" + "\x06reason\x18\x04 \x01(\tR\x06reason\x12\x18\n" + - "\amessage\x18\x05 \x01(\tR\amessage\"#\n" + + "\amessage\x18\x05 \x01(\tR\amessage\"[\n" + "\rGetGpuRequest\x12\x12\n" + - "\x04name\x18\x01 \x01(\tR\x04name\"C\n" + - "\x0eGetGpuResponse\x121\n" + - "\x03gpu\x18\x01 \x01(\v2\x1f.nvidia.nvsentinel.v1alpha1.GpuR\x03gpu\"\x11\n" + - "\x0fListGpusRequest\"R\n" + - "\x10ListGpusResponse\x12>\n" + - "\bgpu_list\x18\x01 \x01(\v2#.nvidia.nvsentinel.v1alpha1.GpuListR\agpuList\"\x12\n" + - "\x10WatchGpusRequest\"`\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x126\n" + + "\x04opts\x18\x02 \x01(\v2\".nvidia.device.v1alpha1.GetOptionsR\x04opts\"?\n" + + "\x0eGetGpuResponse\x12-\n" + + "\x03gpu\x18\x01 \x01(\v2\x1b.nvidia.device.v1alpha1.GpuR\x03gpu\"J\n" + + "\x0fListGpusRequest\x127\n" + + "\x04opts\x18\x01 \x01(\v2#.nvidia.device.v1alpha1.ListOptionsR\x04opts\"N\n" + + "\x10ListGpusResponse\x12:\n" + + "\bgpu_list\x18\x01 \x01(\v2\x1f.nvidia.device.v1alpha1.GpuListR\agpuList\"K\n" + + "\x10WatchGpusRequest\x127\n" + + "\x04opts\x18\x01 \x01(\v2#.nvidia.device.v1alpha1.ListOptionsR\x04opts\"\\\n" + "\x11WatchGpusResponse\x12\x12\n" + - "\x04type\x18\x01 \x01(\tR\x04type\x127\n" + - "\x06object\x18\x02 \x01(\v2\x1f.nvidia.nvsentinel.v1alpha1.GpuR\x06object2\xc0\x02\n" + + "\x04type\x18\x01 \x01(\tR\x04type\x123\n" + + "\x06object\x18\x02 \x01(\v2\x1b.nvidia.device.v1alpha1.GpuR\x06object\"A\n" + + "\x10CreateGpuRequest\x12-\n" + + "\x03gpu\x18\x01 \x01(\v2\x1b.nvidia.device.v1alpha1.GpuR\x03gpu\"\\\n" + + "\x11CreateGpuResponse\x12-\n" + + "\x03gpu\x18\x01 \x01(\v2\x1b.nvidia.device.v1alpha1.GpuR\x03gpu\x12\x18\n" + + "\acreated\x18\x02 \x01(\bR\acreated\"A\n" + + "\x10UpdateGpuRequest\x12-\n" + + "\x03gpu\x18\x01 \x01(\v2\x1b.nvidia.device.v1alpha1.GpuR\x03gpu\"B\n" + + "\x11UpdateGpuResponse\x12-\n" + + "\x03gpu\x18\x01 \x01(\v2\x1b.nvidia.device.v1alpha1.GpuR\x03gpu\"\x92\x01\n" + + "\x16UpdateGpuStatusRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x129\n" + + "\x06status\x18\x02 \x01(\v2!.nvidia.device.v1alpha1.GpuStatusR\x06status\x12)\n" + + "\x10resource_version\x18\x03 \x01(\tR\x0fresourceVersion\"H\n" + + "\x17UpdateGpuStatusResponse\x12-\n" + + "\x03gpu\x18\x01 \x01(\v2\x1b.nvidia.device.v1alpha1.GpuR\x03gpu\"&\n" + + "\x10DeleteGpuRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name2\xaf\x05\n" + "\n" + - "GpuService\x12_\n" + - "\x06GetGpu\x12).nvidia.nvsentinel.v1alpha1.GetGpuRequest\x1a*.nvidia.nvsentinel.v1alpha1.GetGpuResponse\x12e\n" + - "\bListGpus\x12+.nvidia.nvsentinel.v1alpha1.ListGpusRequest\x1a,.nvidia.nvsentinel.v1alpha1.ListGpusResponse\x12j\n" + - "\tWatchGpus\x12,.nvidia.nvsentinel.v1alpha1.WatchGpusRequest\x1a-.nvidia.nvsentinel.v1alpha1.WatchGpusResponse0\x01BBZ@github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1;v1alpha1b\x06proto3" + "GpuService\x12W\n" + + "\x06GetGpu\x12%.nvidia.device.v1alpha1.GetGpuRequest\x1a&.nvidia.device.v1alpha1.GetGpuResponse\x12]\n" + + "\bListGpus\x12'.nvidia.device.v1alpha1.ListGpusRequest\x1a(.nvidia.device.v1alpha1.ListGpusResponse\x12b\n" + + "\tWatchGpus\x12(.nvidia.device.v1alpha1.WatchGpusRequest\x1a).nvidia.device.v1alpha1.WatchGpusResponse0\x01\x12`\n" + + "\tCreateGpu\x12(.nvidia.device.v1alpha1.CreateGpuRequest\x1a).nvidia.device.v1alpha1.CreateGpuResponse\x12`\n" + + "\tUpdateGpu\x12(.nvidia.device.v1alpha1.UpdateGpuRequest\x1a).nvidia.device.v1alpha1.UpdateGpuResponse\x12r\n" + + "\x0fUpdateGpuStatus\x12..nvidia.device.v1alpha1.UpdateGpuStatusRequest\x1a/.nvidia.device.v1alpha1.UpdateGpuStatusResponse\x12M\n" + + "\tDeleteGpu\x12(.nvidia.device.v1alpha1.DeleteGpuRequest\x1a\x16.google.protobuf.EmptyBBZ@github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1;v1alpha1b\x06proto3" var ( file_device_v1alpha1_gpu_proto_rawDescOnce sync.Once @@ -677,41 +1351,72 @@ func file_device_v1alpha1_gpu_proto_rawDescGZIP() []byte { return file_device_v1alpha1_gpu_proto_rawDescData } -var file_device_v1alpha1_gpu_proto_msgTypes = make([]protoimpl.MessageInfo, 11) +var file_device_v1alpha1_gpu_proto_msgTypes = make([]protoimpl.MessageInfo, 22) var file_device_v1alpha1_gpu_proto_goTypes = []any{ - (*Gpu)(nil), // 0: nvidia.nvsentinel.v1alpha1.Gpu - (*GpuList)(nil), // 1: nvidia.nvsentinel.v1alpha1.GpuList - (*GpuSpec)(nil), // 2: nvidia.nvsentinel.v1alpha1.GpuSpec - (*GpuStatus)(nil), // 3: nvidia.nvsentinel.v1alpha1.GpuStatus - (*Condition)(nil), // 4: nvidia.nvsentinel.v1alpha1.Condition - (*GetGpuRequest)(nil), // 5: nvidia.nvsentinel.v1alpha1.GetGpuRequest - (*GetGpuResponse)(nil), // 6: nvidia.nvsentinel.v1alpha1.GetGpuResponse - (*ListGpusRequest)(nil), // 7: nvidia.nvsentinel.v1alpha1.ListGpusRequest - (*ListGpusResponse)(nil), // 8: nvidia.nvsentinel.v1alpha1.ListGpusResponse - (*WatchGpusRequest)(nil), // 9: nvidia.nvsentinel.v1alpha1.WatchGpusRequest - (*WatchGpusResponse)(nil), // 10: nvidia.nvsentinel.v1alpha1.WatchGpusResponse - (*timestamppb.Timestamp)(nil), // 11: google.protobuf.Timestamp + (*ObjectMeta)(nil), // 0: nvidia.device.v1alpha1.ObjectMeta + (*ListMeta)(nil), // 1: nvidia.device.v1alpha1.ListMeta + (*GetOptions)(nil), // 2: nvidia.device.v1alpha1.GetOptions + (*ListOptions)(nil), // 3: nvidia.device.v1alpha1.ListOptions + (*Gpu)(nil), // 4: nvidia.device.v1alpha1.Gpu + (*GpuList)(nil), // 5: nvidia.device.v1alpha1.GpuList + (*GpuSpec)(nil), // 6: nvidia.device.v1alpha1.GpuSpec + (*GpuStatus)(nil), // 7: nvidia.device.v1alpha1.GpuStatus + (*Condition)(nil), // 8: nvidia.device.v1alpha1.Condition + (*GetGpuRequest)(nil), // 9: nvidia.device.v1alpha1.GetGpuRequest + (*GetGpuResponse)(nil), // 10: nvidia.device.v1alpha1.GetGpuResponse + (*ListGpusRequest)(nil), // 11: nvidia.device.v1alpha1.ListGpusRequest + (*ListGpusResponse)(nil), // 12: nvidia.device.v1alpha1.ListGpusResponse + (*WatchGpusRequest)(nil), // 13: nvidia.device.v1alpha1.WatchGpusRequest + (*WatchGpusResponse)(nil), // 14: nvidia.device.v1alpha1.WatchGpusResponse + (*CreateGpuRequest)(nil), // 15: nvidia.device.v1alpha1.CreateGpuRequest + (*CreateGpuResponse)(nil), // 16: nvidia.device.v1alpha1.CreateGpuResponse + (*UpdateGpuRequest)(nil), // 17: nvidia.device.v1alpha1.UpdateGpuRequest + (*UpdateGpuResponse)(nil), // 18: nvidia.device.v1alpha1.UpdateGpuResponse + (*UpdateGpuStatusRequest)(nil), // 19: nvidia.device.v1alpha1.UpdateGpuStatusRequest + (*UpdateGpuStatusResponse)(nil), // 20: nvidia.device.v1alpha1.UpdateGpuStatusResponse + (*DeleteGpuRequest)(nil), // 21: nvidia.device.v1alpha1.DeleteGpuRequest + (*timestamppb.Timestamp)(nil), // 22: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 23: google.protobuf.Empty } var file_device_v1alpha1_gpu_proto_depIdxs = []int32{ - 2, // 0: nvidia.nvsentinel.v1alpha1.Gpu.spec:type_name -> nvidia.nvsentinel.v1alpha1.GpuSpec - 3, // 1: nvidia.nvsentinel.v1alpha1.Gpu.status:type_name -> nvidia.nvsentinel.v1alpha1.GpuStatus - 0, // 2: nvidia.nvsentinel.v1alpha1.GpuList.items:type_name -> nvidia.nvsentinel.v1alpha1.Gpu - 4, // 3: nvidia.nvsentinel.v1alpha1.GpuStatus.conditions:type_name -> nvidia.nvsentinel.v1alpha1.Condition - 11, // 4: nvidia.nvsentinel.v1alpha1.Condition.last_transition_time:type_name -> google.protobuf.Timestamp - 0, // 5: nvidia.nvsentinel.v1alpha1.GetGpuResponse.gpu:type_name -> nvidia.nvsentinel.v1alpha1.Gpu - 1, // 6: nvidia.nvsentinel.v1alpha1.ListGpusResponse.gpu_list:type_name -> nvidia.nvsentinel.v1alpha1.GpuList - 0, // 7: nvidia.nvsentinel.v1alpha1.WatchGpusResponse.object:type_name -> nvidia.nvsentinel.v1alpha1.Gpu - 5, // 8: nvidia.nvsentinel.v1alpha1.GpuService.GetGpu:input_type -> nvidia.nvsentinel.v1alpha1.GetGpuRequest - 7, // 9: nvidia.nvsentinel.v1alpha1.GpuService.ListGpus:input_type -> nvidia.nvsentinel.v1alpha1.ListGpusRequest - 9, // 10: nvidia.nvsentinel.v1alpha1.GpuService.WatchGpus:input_type -> nvidia.nvsentinel.v1alpha1.WatchGpusRequest - 6, // 11: nvidia.nvsentinel.v1alpha1.GpuService.GetGpu:output_type -> nvidia.nvsentinel.v1alpha1.GetGpuResponse - 8, // 12: nvidia.nvsentinel.v1alpha1.GpuService.ListGpus:output_type -> nvidia.nvsentinel.v1alpha1.ListGpusResponse - 10, // 13: nvidia.nvsentinel.v1alpha1.GpuService.WatchGpus:output_type -> nvidia.nvsentinel.v1alpha1.WatchGpusResponse - 11, // [11:14] is the sub-list for method output_type - 8, // [8:11] is the sub-list for method input_type - 8, // [8:8] is the sub-list for extension type_name - 8, // [8:8] is the sub-list for extension extendee - 0, // [0:8] is the sub-list for field type_name + 0, // 0: nvidia.device.v1alpha1.Gpu.metadata:type_name -> nvidia.device.v1alpha1.ObjectMeta + 6, // 1: nvidia.device.v1alpha1.Gpu.spec:type_name -> nvidia.device.v1alpha1.GpuSpec + 7, // 2: nvidia.device.v1alpha1.Gpu.status:type_name -> nvidia.device.v1alpha1.GpuStatus + 1, // 3: nvidia.device.v1alpha1.GpuList.metadata:type_name -> nvidia.device.v1alpha1.ListMeta + 4, // 4: nvidia.device.v1alpha1.GpuList.items:type_name -> nvidia.device.v1alpha1.Gpu + 8, // 5: nvidia.device.v1alpha1.GpuStatus.conditions:type_name -> nvidia.device.v1alpha1.Condition + 22, // 6: nvidia.device.v1alpha1.Condition.last_transition_time:type_name -> google.protobuf.Timestamp + 2, // 7: nvidia.device.v1alpha1.GetGpuRequest.opts:type_name -> nvidia.device.v1alpha1.GetOptions + 4, // 8: nvidia.device.v1alpha1.GetGpuResponse.gpu:type_name -> nvidia.device.v1alpha1.Gpu + 3, // 9: nvidia.device.v1alpha1.ListGpusRequest.opts:type_name -> nvidia.device.v1alpha1.ListOptions + 5, // 10: nvidia.device.v1alpha1.ListGpusResponse.gpu_list:type_name -> nvidia.device.v1alpha1.GpuList + 3, // 11: nvidia.device.v1alpha1.WatchGpusRequest.opts:type_name -> nvidia.device.v1alpha1.ListOptions + 4, // 12: nvidia.device.v1alpha1.WatchGpusResponse.object:type_name -> nvidia.device.v1alpha1.Gpu + 4, // 13: nvidia.device.v1alpha1.CreateGpuRequest.gpu:type_name -> nvidia.device.v1alpha1.Gpu + 4, // 14: nvidia.device.v1alpha1.CreateGpuResponse.gpu:type_name -> nvidia.device.v1alpha1.Gpu + 4, // 15: nvidia.device.v1alpha1.UpdateGpuRequest.gpu:type_name -> nvidia.device.v1alpha1.Gpu + 4, // 16: nvidia.device.v1alpha1.UpdateGpuResponse.gpu:type_name -> nvidia.device.v1alpha1.Gpu + 7, // 17: nvidia.device.v1alpha1.UpdateGpuStatusRequest.status:type_name -> nvidia.device.v1alpha1.GpuStatus + 4, // 18: nvidia.device.v1alpha1.UpdateGpuStatusResponse.gpu:type_name -> nvidia.device.v1alpha1.Gpu + 9, // 19: nvidia.device.v1alpha1.GpuService.GetGpu:input_type -> nvidia.device.v1alpha1.GetGpuRequest + 11, // 20: nvidia.device.v1alpha1.GpuService.ListGpus:input_type -> nvidia.device.v1alpha1.ListGpusRequest + 13, // 21: nvidia.device.v1alpha1.GpuService.WatchGpus:input_type -> nvidia.device.v1alpha1.WatchGpusRequest + 15, // 22: nvidia.device.v1alpha1.GpuService.CreateGpu:input_type -> nvidia.device.v1alpha1.CreateGpuRequest + 17, // 23: nvidia.device.v1alpha1.GpuService.UpdateGpu:input_type -> nvidia.device.v1alpha1.UpdateGpuRequest + 19, // 24: nvidia.device.v1alpha1.GpuService.UpdateGpuStatus:input_type -> nvidia.device.v1alpha1.UpdateGpuStatusRequest + 21, // 25: nvidia.device.v1alpha1.GpuService.DeleteGpu:input_type -> nvidia.device.v1alpha1.DeleteGpuRequest + 10, // 26: nvidia.device.v1alpha1.GpuService.GetGpu:output_type -> nvidia.device.v1alpha1.GetGpuResponse + 12, // 27: nvidia.device.v1alpha1.GpuService.ListGpus:output_type -> nvidia.device.v1alpha1.ListGpusResponse + 14, // 28: nvidia.device.v1alpha1.GpuService.WatchGpus:output_type -> nvidia.device.v1alpha1.WatchGpusResponse + 16, // 29: nvidia.device.v1alpha1.GpuService.CreateGpu:output_type -> nvidia.device.v1alpha1.CreateGpuResponse + 18, // 30: nvidia.device.v1alpha1.GpuService.UpdateGpu:output_type -> nvidia.device.v1alpha1.UpdateGpuResponse + 20, // 31: nvidia.device.v1alpha1.GpuService.UpdateGpuStatus:output_type -> nvidia.device.v1alpha1.UpdateGpuStatusResponse + 23, // 32: nvidia.device.v1alpha1.GpuService.DeleteGpu:output_type -> google.protobuf.Empty + 26, // [26:33] is the sub-list for method output_type + 19, // [19:26] is the sub-list for method input_type + 19, // [19:19] is the sub-list for extension type_name + 19, // [19:19] is the sub-list for extension extendee + 0, // [0:19] is the sub-list for field type_name } func init() { file_device_v1alpha1_gpu_proto_init() } @@ -725,7 +1430,7 @@ func file_device_v1alpha1_gpu_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_device_v1alpha1_gpu_proto_rawDesc), len(file_device_v1alpha1_gpu_proto_rawDesc)), NumEnums: 0, - NumMessages: 11, + NumMessages: 22, NumExtensions: 0, NumServices: 1, }, diff --git a/api/gen/go/device/v1alpha1/gpu_grpc.pb.go b/api/gen/go/device/v1alpha1/gpu_grpc.pb.go index 1ebc7526d..5c48ad741 100644 --- a/api/gen/go/device/v1alpha1/gpu_grpc.pb.go +++ b/api/gen/go/device/v1alpha1/gpu_grpc.pb.go @@ -25,6 +25,7 @@ import ( grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" + emptypb "google.golang.org/protobuf/types/known/emptypb" ) // This is a compile-time assertion to ensure that this generated file @@ -33,23 +34,116 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - GpuService_GetGpu_FullMethodName = "/nvidia.nvsentinel.v1alpha1.GpuService/GetGpu" - GpuService_ListGpus_FullMethodName = "/nvidia.nvsentinel.v1alpha1.GpuService/ListGpus" - GpuService_WatchGpus_FullMethodName = "/nvidia.nvsentinel.v1alpha1.GpuService/WatchGpus" + GpuService_GetGpu_FullMethodName = "/nvidia.device.v1alpha1.GpuService/GetGpu" + GpuService_ListGpus_FullMethodName = "/nvidia.device.v1alpha1.GpuService/ListGpus" + GpuService_WatchGpus_FullMethodName = "/nvidia.device.v1alpha1.GpuService/WatchGpus" + GpuService_CreateGpu_FullMethodName = "/nvidia.device.v1alpha1.GpuService/CreateGpu" + GpuService_UpdateGpu_FullMethodName = "/nvidia.device.v1alpha1.GpuService/UpdateGpu" + GpuService_UpdateGpuStatus_FullMethodName = "/nvidia.device.v1alpha1.GpuService/UpdateGpuStatus" + GpuService_DeleteGpu_FullMethodName = "/nvidia.device.v1alpha1.GpuService/DeleteGpu" ) // GpuServiceClient is the client API for GpuService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. // -// GpuService provides an API for observing GPU resources. +// GpuService provides a unified API for managing GPU resources. +// +// Read operations (Get, List, Watch) are intended for consumers like +// device plugins and DRA drivers. +// +// Write operations (Create, Update, UpdateStatus, Delete) are intended +// for providers like health monitors. +// +// All write operations acquire exclusive write locks, blocking ALL consumer +// read operations until the write completes. This ensures consumers never +// read stale data during status transitions. +// +// Access control should be enforced at the infrastructure level (e.g., +// NetworkPolicy, mTLS) rather than in the API itself. type GpuServiceClient interface { // GetGpu retrieves a single GPU resource by its unique name. + // + // Returns: + // - The GPU if found + // - NOT_FOUND if no GPU with that name exists GetGpu(ctx context.Context, in *GetGpuRequest, opts ...grpc.CallOption) (*GetGpuResponse, error) - // ListGpus retrieves a list of GPU resources. + // ListGpus retrieves all GPU resources. + // + // Returns a GpuList containing all registered GPUs. ListGpus(ctx context.Context, in *ListGpusRequest, opts ...grpc.CallOption) (*ListGpusResponse, error) // WatchGpus streams lifecycle events for GPU resources. + // + // Events are sent for: + // - ADDED: A new GPU was created + // - MODIFIED: An existing GPU was updated + // - DELETED: A GPU was removed + // + // The stream will receive an initial batch of ADDED events for all + // existing GPUs, followed by real-time events as changes occur. WatchGpus(ctx context.Context, in *WatchGpusRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[WatchGpusResponse], error) + // CreateGpu registers a new GPU with the server. + // + // The GPU name (metadata.name) must be unique. This operation uses + // idempotent semantics: + // - If the GPU doesn't exist, it is created and created=true is returned + // - If a GPU with the same name exists, the existing GPU is returned + // with created=false (no error is returned) + // + // This enables safe retry patterns where providers can call CreateGpu + // without needing to handle ALREADY_EXISTS errors. + // + // This operation acquires a write lock, blocking consumer reads + // until the operation completes. + // + // Returns: + // - The GPU (created or existing) with server-assigned fields + // - INVALID_ARGUMENT if required fields are missing + CreateGpu(ctx context.Context, in *CreateGpuRequest, opts ...grpc.CallOption) (*CreateGpuResponse, error) + // UpdateGpu replaces an existing GPU resource. + // + // The entire GPU (spec and status) is replaced. Use UpdateGpuStatus + // if you only need to update the status. + // + // Optimistic concurrency: If resource_version is provided and does + // not match the current version, returns ABORTED. + // + // This operation acquires a write lock. + // + // Returns: + // - The updated GPU + // - NOT_FOUND if the GPU doesn't exist + // - ABORTED if resource_version doesn't match (conflict) + UpdateGpu(ctx context.Context, in *UpdateGpuRequest, opts ...grpc.CallOption) (*UpdateGpuResponse, error) + // UpdateGpuStatus updates only the status of an existing GPU. + // + // This is the primary method for health monitors to report GPU state. + // The spec is not modified. + // + // This follows the Kubernetes subresource pattern where status is + // updated separately from spec. + // + // This operation acquires a write lock, blocking consumer reads + // until the operation completes. This ensures consumers never read + // stale data during status transitions (e.g., healthy → unhealthy). + // + // Returns: + // - The updated GPU + // - NOT_FOUND if the GPU doesn't exist + // - ABORTED if resource_version doesn't match (conflict) + UpdateGpuStatus(ctx context.Context, in *UpdateGpuStatusRequest, opts ...grpc.CallOption) (*UpdateGpuStatusResponse, error) + // DeleteGpu removes a GPU from the server. + // + // After deletion, the GPU will no longer appear in ListGpus or + // GetGpu responses. Active WatchGpus streams will receive a + // DELETED event. + // + // This operation acquires a write lock. + // + // Returns: + // - Empty on success + // - NOT_FOUND if the GPU doesn't exist + DeleteGpu(ctx context.Context, in *DeleteGpuRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) } type gpuServiceClient struct { @@ -99,18 +193,147 @@ func (c *gpuServiceClient) WatchGpus(ctx context.Context, in *WatchGpusRequest, // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. type GpuService_WatchGpusClient = grpc.ServerStreamingClient[WatchGpusResponse] +func (c *gpuServiceClient) CreateGpu(ctx context.Context, in *CreateGpuRequest, opts ...grpc.CallOption) (*CreateGpuResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(CreateGpuResponse) + err := c.cc.Invoke(ctx, GpuService_CreateGpu_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *gpuServiceClient) UpdateGpu(ctx context.Context, in *UpdateGpuRequest, opts ...grpc.CallOption) (*UpdateGpuResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(UpdateGpuResponse) + err := c.cc.Invoke(ctx, GpuService_UpdateGpu_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *gpuServiceClient) UpdateGpuStatus(ctx context.Context, in *UpdateGpuStatusRequest, opts ...grpc.CallOption) (*UpdateGpuStatusResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(UpdateGpuStatusResponse) + err := c.cc.Invoke(ctx, GpuService_UpdateGpuStatus_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *gpuServiceClient) DeleteGpu(ctx context.Context, in *DeleteGpuRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, GpuService_DeleteGpu_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // GpuServiceServer is the server API for GpuService service. // All implementations must embed UnimplementedGpuServiceServer // for forward compatibility. // -// GpuService provides an API for observing GPU resources. +// GpuService provides a unified API for managing GPU resources. +// +// Read operations (Get, List, Watch) are intended for consumers like +// device plugins and DRA drivers. +// +// Write operations (Create, Update, UpdateStatus, Delete) are intended +// for providers like health monitors. +// +// All write operations acquire exclusive write locks, blocking ALL consumer +// read operations until the write completes. This ensures consumers never +// read stale data during status transitions. +// +// Access control should be enforced at the infrastructure level (e.g., +// NetworkPolicy, mTLS) rather than in the API itself. type GpuServiceServer interface { // GetGpu retrieves a single GPU resource by its unique name. + // + // Returns: + // - The GPU if found + // - NOT_FOUND if no GPU with that name exists GetGpu(context.Context, *GetGpuRequest) (*GetGpuResponse, error) - // ListGpus retrieves a list of GPU resources. + // ListGpus retrieves all GPU resources. + // + // Returns a GpuList containing all registered GPUs. ListGpus(context.Context, *ListGpusRequest) (*ListGpusResponse, error) // WatchGpus streams lifecycle events for GPU resources. + // + // Events are sent for: + // - ADDED: A new GPU was created + // - MODIFIED: An existing GPU was updated + // - DELETED: A GPU was removed + // + // The stream will receive an initial batch of ADDED events for all + // existing GPUs, followed by real-time events as changes occur. WatchGpus(*WatchGpusRequest, grpc.ServerStreamingServer[WatchGpusResponse]) error + // CreateGpu registers a new GPU with the server. + // + // The GPU name (metadata.name) must be unique. This operation uses + // idempotent semantics: + // - If the GPU doesn't exist, it is created and created=true is returned + // - If a GPU with the same name exists, the existing GPU is returned + // with created=false (no error is returned) + // + // This enables safe retry patterns where providers can call CreateGpu + // without needing to handle ALREADY_EXISTS errors. + // + // This operation acquires a write lock, blocking consumer reads + // until the operation completes. + // + // Returns: + // - The GPU (created or existing) with server-assigned fields + // - INVALID_ARGUMENT if required fields are missing + CreateGpu(context.Context, *CreateGpuRequest) (*CreateGpuResponse, error) + // UpdateGpu replaces an existing GPU resource. + // + // The entire GPU (spec and status) is replaced. Use UpdateGpuStatus + // if you only need to update the status. + // + // Optimistic concurrency: If resource_version is provided and does + // not match the current version, returns ABORTED. + // + // This operation acquires a write lock. + // + // Returns: + // - The updated GPU + // - NOT_FOUND if the GPU doesn't exist + // - ABORTED if resource_version doesn't match (conflict) + UpdateGpu(context.Context, *UpdateGpuRequest) (*UpdateGpuResponse, error) + // UpdateGpuStatus updates only the status of an existing GPU. + // + // This is the primary method for health monitors to report GPU state. + // The spec is not modified. + // + // This follows the Kubernetes subresource pattern where status is + // updated separately from spec. + // + // This operation acquires a write lock, blocking consumer reads + // until the operation completes. This ensures consumers never read + // stale data during status transitions (e.g., healthy → unhealthy). + // + // Returns: + // - The updated GPU + // - NOT_FOUND if the GPU doesn't exist + // - ABORTED if resource_version doesn't match (conflict) + UpdateGpuStatus(context.Context, *UpdateGpuStatusRequest) (*UpdateGpuStatusResponse, error) + // DeleteGpu removes a GPU from the server. + // + // After deletion, the GPU will no longer appear in ListGpus or + // GetGpu responses. Active WatchGpus streams will receive a + // DELETED event. + // + // This operation acquires a write lock. + // + // Returns: + // - Empty on success + // - NOT_FOUND if the GPU doesn't exist + DeleteGpu(context.Context, *DeleteGpuRequest) (*emptypb.Empty, error) mustEmbedUnimplementedGpuServiceServer() } @@ -130,6 +353,18 @@ func (UnimplementedGpuServiceServer) ListGpus(context.Context, *ListGpusRequest) func (UnimplementedGpuServiceServer) WatchGpus(*WatchGpusRequest, grpc.ServerStreamingServer[WatchGpusResponse]) error { return status.Errorf(codes.Unimplemented, "method WatchGpus not implemented") } +func (UnimplementedGpuServiceServer) CreateGpu(context.Context, *CreateGpuRequest) (*CreateGpuResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CreateGpu not implemented") +} +func (UnimplementedGpuServiceServer) UpdateGpu(context.Context, *UpdateGpuRequest) (*UpdateGpuResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method UpdateGpu not implemented") +} +func (UnimplementedGpuServiceServer) UpdateGpuStatus(context.Context, *UpdateGpuStatusRequest) (*UpdateGpuStatusResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method UpdateGpuStatus not implemented") +} +func (UnimplementedGpuServiceServer) DeleteGpu(context.Context, *DeleteGpuRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteGpu not implemented") +} func (UnimplementedGpuServiceServer) mustEmbedUnimplementedGpuServiceServer() {} func (UnimplementedGpuServiceServer) testEmbeddedByValue() {} @@ -198,11 +433,83 @@ func _GpuService_WatchGpus_Handler(srv interface{}, stream grpc.ServerStream) er // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. type GpuService_WatchGpusServer = grpc.ServerStreamingServer[WatchGpusResponse] +func _GpuService_CreateGpu_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateGpuRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GpuServiceServer).CreateGpu(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: GpuService_CreateGpu_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GpuServiceServer).CreateGpu(ctx, req.(*CreateGpuRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _GpuService_UpdateGpu_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(UpdateGpuRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GpuServiceServer).UpdateGpu(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: GpuService_UpdateGpu_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GpuServiceServer).UpdateGpu(ctx, req.(*UpdateGpuRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _GpuService_UpdateGpuStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(UpdateGpuStatusRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GpuServiceServer).UpdateGpuStatus(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: GpuService_UpdateGpuStatus_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GpuServiceServer).UpdateGpuStatus(ctx, req.(*UpdateGpuStatusRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _GpuService_DeleteGpu_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteGpuRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GpuServiceServer).DeleteGpu(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: GpuService_DeleteGpu_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GpuServiceServer).DeleteGpu(ctx, req.(*DeleteGpuRequest)) + } + return interceptor(ctx, in, info, handler) +} + // GpuService_ServiceDesc is the grpc.ServiceDesc for GpuService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var GpuService_ServiceDesc = grpc.ServiceDesc{ - ServiceName: "nvidia.nvsentinel.v1alpha1.GpuService", + ServiceName: "nvidia.device.v1alpha1.GpuService", HandlerType: (*GpuServiceServer)(nil), Methods: []grpc.MethodDesc{ { @@ -213,6 +520,22 @@ var GpuService_ServiceDesc = grpc.ServiceDesc{ MethodName: "ListGpus", Handler: _GpuService_ListGpus_Handler, }, + { + MethodName: "CreateGpu", + Handler: _GpuService_CreateGpu_Handler, + }, + { + MethodName: "UpdateGpu", + Handler: _GpuService_UpdateGpu_Handler, + }, + { + MethodName: "UpdateGpuStatus", + Handler: _GpuService_UpdateGpuStatus_Handler, + }, + { + MethodName: "DeleteGpu", + Handler: _GpuService_DeleteGpu_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/api/go.mod b/api/go.mod index c99562dcb..ba907b5bf 100644 --- a/api/go.mod +++ b/api/go.mod @@ -1,10 +1,11 @@ module github.com/nvidia/nvsentinel/api -go 1.25 +go 1.25.0 require ( - google.golang.org/grpc v1.78.0 - google.golang.org/protobuf v1.36.11 + github.com/google/go-cmp v0.7.0 + google.golang.org/grpc v1.77.0 + google.golang.org/protobuf v1.36.10 ) require ( @@ -12,4 +13,22 @@ require ( golang.org/x/sys v0.38.0 // indirect golang.org/x/text v0.31.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251124214823-79d6a2a48846 // indirect + k8s.io/apimachinery v0.35.0 +) + +require ( + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/x448/float16 v0.8.4 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect ) diff --git a/api/go.sum b/api/go.sum index f3fc747f6..0d01299d3 100644 --- a/api/go.sum +++ b/api/go.sum @@ -1,3 +1,8 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -6,8 +11,27 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= @@ -20,6 +44,8 @@ go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6 go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= @@ -30,7 +56,29 @@ gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/rpc v0.0.0-20251124214823-79d6a2a48846 h1:Wgl1rcDNThT+Zn47YyCXOXyX/COgMTIdhJ717F0l4xk= google.golang.org/genproto/googleapis/rpc v0.0.0-20251124214823-79d6a2a48846/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= -google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= -google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM= +google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/apimachinery v0.35.0 h1:Z2L3IHvPVv/MJ7xRxHEtk6GoJElaAqDCCU0S6ncYok8= +k8s.io/apimachinery v0.35.0/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/api/hack/boilerplate.go.txt b/api/hack/boilerplate.go.txt new file mode 100644 index 000000000..e1732e8d5 --- /dev/null +++ b/api/hack/boilerplate.go.txt @@ -0,0 +1,13 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. diff --git a/api/hack/update-codegen.sh b/api/hack/update-codegen.sh new file mode 100755 index 000000000..921ee1574 --- /dev/null +++ b/api/hack/update-codegen.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)" +CODEGEN_ROOT="${REPO_ROOT}/code-generator" + +export KUBE_CODEGEN_ROOT="${CODEGEN_ROOT}" + +source "${CODEGEN_ROOT}/kube_codegen.sh" + +kube::codegen::gen_proto_bindings \ + --output-dir "gen/go" \ + --proto-root "proto" \ + "${REPO_ROOT}/api" + +kube::codegen::gen_helpers \ + --boilerplate "${REPO_ROOT}/api/hack/boilerplate.go.txt" \ + "${REPO_ROOT}/api" diff --git a/api/nvsentinel/v1alpha1/doc.go b/api/nvsentinel/v1alpha1/doc.go new file mode 100644 index 000000000..c74370b8b --- /dev/null +++ b/api/nvsentinel/v1alpha1/doc.go @@ -0,0 +1,21 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +k8s:deepcopy-gen=package +// +groupName=nvsentinel.nvidia.com + +// Package v1alpha1 contains the API Schema definitions for the nvsentinel.nvidia.com v1alpha1 API group. +// This package includes types for HealthEvent CRD used to coordinate fault-handling across +// NVSentinel components (fault-quarantine, node-drainer, remediation controllers). +package v1alpha1 diff --git a/api/nvsentinel/v1alpha1/groupversion_info.go b/api/nvsentinel/v1alpha1/groupversion_info.go new file mode 100644 index 000000000..9e32d3132 --- /dev/null +++ b/api/nvsentinel/v1alpha1/groupversion_info.go @@ -0,0 +1,48 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// SchemeGroupVersion is group version used to register these objects. +var SchemeGroupVersion = schema.GroupVersion{Group: "nvsentinel.nvidia.com", Version: "v1alpha1"} + +// Resource takes an unqualified resource and returns a Group qualified GroupResource. +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} + +var ( + // SchemeBuilder initializes a scheme builder. + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) + +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &HealthEvent{}, + &HealthEventList{}, + ) + + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + + return nil +} diff --git a/api/nvsentinel/v1alpha1/healthevent_types.go b/api/nvsentinel/v1alpha1/healthevent_types.go new file mode 100644 index 000000000..97089f1fd --- /dev/null +++ b/api/nvsentinel/v1alpha1/healthevent_types.go @@ -0,0 +1,342 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ============================================================================= +// HealthEvent CRD +// ============================================================================= + +// HealthEvent represents a GPU health event detected by a health monitor. +// It replaces the MongoDB HealthEventWithStatus document and enables +// Kubernetes-native coordination between fault-quarantine, node-drainer, +// and remediation controllers. +// +// +genclient +// +genclient:nonNamespaced +// +k8s:deepcopy-gen=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster,shortName=he;hevt +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Node",type=string,JSONPath=`.spec.nodeName` +// +kubebuilder:printcolumn:name="Source",type=string,JSONPath=`.spec.source` +// +kubebuilder:printcolumn:name="Fatal",type=boolean,JSONPath=`.spec.isFatal` +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` +type HealthEvent struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec HealthEventSpec `json:"spec,omitempty"` + Status HealthEventStatus `json:"status,omitempty"` +} + +// HealthEventList contains a list of HealthEvent resources. +// +// +k8s:deepcopy-gen=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true +type HealthEventList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []HealthEvent `json:"items"` +} + +// ============================================================================= +// Spec (Immutable - what happened) +// ============================================================================= + +// HealthEventSpec defines the immutable details of a health event. +// These fields are set when the event is created and should not be modified. +// +// +k8s:deepcopy-gen=true +type HealthEventSpec struct { + // Source identifies the health monitor that detected this event. + // Examples: "nvml-health-monitor", "dcgm", "csp-health-monitor", "syslog-monitor" + // +kubebuilder:validation:Required + Source string `json:"source"` + + // Agent is the name of the agent that reported the event (for compatibility with protobuf). + // +optional + Agent string `json:"agent,omitempty"` + + // NodeName is the Kubernetes node where the event occurred. + // +kubebuilder:validation:Required + NodeName string `json:"nodeName"` + + // ComponentClass identifies the type of component affected. + // Examples: "GPU", "NIC", "Storage", "Memory" + // +kubebuilder:validation:Required + ComponentClass string `json:"componentClass"` + + // CheckName identifies the specific health check that detected the issue. + // Examples: "xid-error-check", "memory-ecc-check", "thermal-check" + // +kubebuilder:validation:Required + CheckName string `json:"checkName"` + + // IsFatal indicates whether this event requires immediate node quarantine. + // Fatal events trigger the full fault-handling workflow. + // +kubebuilder:default=false + IsFatal bool `json:"isFatal"` + + // IsHealthy indicates the current health state. + // false = unhealthy (problem detected), true = healthy (recovered) + // +kubebuilder:default=false + IsHealthy bool `json:"isHealthy"` + + // Message provides a human-readable description of the event. + // +optional + Message string `json:"message,omitempty"` + + // RecommendedAction suggests what action should be taken. + //nolint:lll // kubebuilder validation must be on single line + // +kubebuilder:validation:Enum=NONE;COMPONENT_RESET;CONTACT_SUPPORT;RUN_FIELDDIAG;RESTART_VM;RESTART_BM;REPLACE_VM;RUN_DCGMEUD;UNKNOWN + // +kubebuilder:default=NONE + RecommendedAction RecommendedAction `json:"recommendedAction,omitempty"` + + // ErrorCodes contains the error codes associated with this event. + // For GPU events, these are typically XID error codes (e.g., "79", "31"). + // +optional + ErrorCodes []string `json:"errorCodes,omitempty"` + + // EntitiesImpacted lists the specific entities affected by this event. + // +optional + EntitiesImpacted []Entity `json:"entitiesImpacted,omitempty"` + + // DetectedAt is the timestamp when the event was first detected. + // +kubebuilder:validation:Required + DetectedAt metav1.Time `json:"detectedAt"` + + // Overrides allows customizing the fault-handling behavior for this event. + // +optional + Overrides *BehaviourOverrides `json:"overrides,omitempty"` + + // Metadata contains additional key-value data about the event. + // Common keys: "uuid", "temperature", "serialNumber" + // +optional + Metadata map[string]string `json:"metadata,omitempty"` + + // Maintenance contains CSP maintenance event details. + // Only populated when Source is "csp-health-monitor". + // +optional + Maintenance *MaintenanceInfo `json:"maintenance,omitempty"` +} + +// ============================================================================= +// Status (Mutable - processing state) +// ============================================================================= + +// HealthEventStatus defines the current processing state of a health event. +// These fields are updated by controllers as the event moves through the workflow. +// +// +k8s:deepcopy-gen=true +type HealthEventStatus struct { + // Phase represents the current state in the fault-handling workflow. + // +kubebuilder:validation:Enum=New;Quarantined;Draining;Drained;Remediating;Remediated;Resolved;Cancelled + // +kubebuilder:default=New + Phase HealthEventPhase `json:"phase,omitempty"` + + // Conditions provide detailed status for each stage of processing. + // +optional + Conditions []HealthEventCondition `json:"conditions,omitempty"` + + // LastRemediationTime is when remediation was last attempted. + // +optional + LastRemediationTime *metav1.Time `json:"lastRemediationTime,omitempty"` + + // ResolvedAt is when the event reached the Resolved phase. + // Used for TTL calculation. + // +optional + ResolvedAt *metav1.Time `json:"resolvedAt,omitempty"` + + // ObservedGeneration is the generation observed by the controller. + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` +} + +// ============================================================================= +// Enums and Types +// ============================================================================= + +// HealthEventPhase represents the current phase of health event processing. +// +kubebuilder:validation:Enum=New;Quarantined;Draining;Drained;Remediating;Remediated;Resolved;Cancelled +type HealthEventPhase string + +const ( + // PhaseNew indicates the event was just created and needs processing. + PhaseNew HealthEventPhase = "New" + + // PhaseQuarantined indicates the node has been cordoned. + PhaseQuarantined HealthEventPhase = "Quarantined" + + // PhaseDraining indicates pods are being evicted from the node. + PhaseDraining HealthEventPhase = "Draining" + + // PhaseDrained indicates all user pods have been evicted. + PhaseDrained HealthEventPhase = "Drained" + + // PhaseRemediating indicates remediation is in progress. + PhaseRemediating HealthEventPhase = "Remediating" + + // PhaseRemediated indicates remediation completed. + PhaseRemediated HealthEventPhase = "Remediated" + + // PhaseResolved indicates the event has been fully handled and GPU is healthy. + PhaseResolved HealthEventPhase = "Resolved" + + // PhaseCancelled indicates the event was cancelled (e.g., override skip=true). + PhaseCancelled HealthEventPhase = "Cancelled" +) + +// RecommendedAction indicates the suggested remediation action. +// +// +kubebuilder:validation:Enum=NONE;COMPONENT_RESET;CONTACT_SUPPORT;RUN_FIELDDIAG;RESTART_VM;RESTART_BM;REPLACE_VM;RUN_DCGMEUD;UNKNOWN +// +//nolint:lll // kubebuilder validation must be on single line +type RecommendedAction string + +const ( + ActionNone RecommendedAction = "NONE" + ActionComponentReset RecommendedAction = "COMPONENT_RESET" + ActionContactSupport RecommendedAction = "CONTACT_SUPPORT" + ActionRunFieldDiag RecommendedAction = "RUN_FIELDDIAG" + ActionRestartVM RecommendedAction = "RESTART_VM" + ActionRestartBM RecommendedAction = "RESTART_BM" + ActionReplaceVM RecommendedAction = "REPLACE_VM" + ActionRunDCGMEUD RecommendedAction = "RUN_DCGMEUD" + ActionUnknown RecommendedAction = "UNKNOWN" +) + +// Entity represents an impacted entity (GPU, NIC, etc.). +type Entity struct { + // Type identifies the kind of entity (e.g., "GPU", "NIC"). + Type string `json:"type"` + + // Value is the identifier for the entity (e.g., GPU UUID). + Value string `json:"value"` +} + +// BehaviourOverrides allows customizing fault-handling behavior. +type BehaviourOverrides struct { + // Quarantine overrides control node cordoning behavior. + // +optional + Quarantine *ActionOverride `json:"quarantine,omitempty"` + + // Drain overrides control pod eviction behavior. + // +optional + Drain *ActionOverride `json:"drain,omitempty"` +} + +// ActionOverride specifies how to override a fault-handling action. +type ActionOverride struct { + // Force causes the action to proceed even if conditions aren't met. + // +kubebuilder:default=false + Force bool `json:"force,omitempty"` + + // Skip causes the action to be skipped entirely. + // +kubebuilder:default=false + Skip bool `json:"skip,omitempty"` +} + +// MaintenanceInfo contains CSP maintenance event details. +// Only populated for events from csp-health-monitor. +type MaintenanceInfo struct { + // CSP identifies the cloud service provider. + // +kubebuilder:validation:Enum=aws;gcp + CSP string `json:"csp"` + + // EventID is the CSP's identifier for this maintenance event. + EventID string `json:"eventId"` + + // MaintenanceType indicates if this is scheduled or unscheduled. + // +kubebuilder:validation:Enum=SCHEDULED;UNSCHEDULED + MaintenanceType string `json:"maintenanceType"` + + // CSPStatus is the status reported by the CSP. + // +optional + CSPStatus string `json:"cspStatus,omitempty"` + + // ScheduledStartTime is when maintenance is scheduled to begin. + // +optional + ScheduledStartTime *metav1.Time `json:"scheduledStartTime,omitempty"` + + // ScheduledEndTime is when maintenance is scheduled to end. + // +optional + ScheduledEndTime *metav1.Time `json:"scheduledEndTime,omitempty"` + + // ActualStartTime is when maintenance actually started. + // +optional + ActualStartTime *metav1.Time `json:"actualStartTime,omitempty"` + + // ActualEndTime is when maintenance actually ended. + // +optional + ActualEndTime *metav1.Time `json:"actualEndTime,omitempty"` +} + +// ============================================================================= +// Conditions +// ============================================================================= + +// HealthEventCondition represents a condition of a HealthEvent. +// Follows the standard Kubernetes condition pattern. +type HealthEventCondition struct { + // Type is the type of condition. + // +kubebuilder:validation:Enum=Detected;NodeQuarantined;PodsDrained;Remediated;Resolved + Type HealthEventConditionType `json:"type"` + + // Status is the status of the condition (True, False, Unknown). + // +kubebuilder:validation:Enum=True;False;Unknown + Status metav1.ConditionStatus `json:"status"` + + // LastTransitionTime is the last time the condition transitioned. + LastTransitionTime metav1.Time `json:"lastTransitionTime"` + + // Reason is a brief machine-readable reason for the condition. + // +optional + Reason string `json:"reason,omitempty"` + + // Message is a human-readable description of the condition. + // +optional + Message string `json:"message,omitempty"` + + // ObservedGeneration is the generation of the resource when this condition was set. + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` +} + +// HealthEventConditionType is the type of HealthEvent condition. +// +kubebuilder:validation:Enum=Detected;NodeQuarantined;PodsDrained;Remediated;Resolved +type HealthEventConditionType string + +const ( + // ConditionDetected indicates the event was detected by a health monitor. + ConditionDetected HealthEventConditionType = "Detected" + + // ConditionNodeQuarantined indicates the node has been cordoned. + ConditionNodeQuarantined HealthEventConditionType = "NodeQuarantined" + + // ConditionPodsDrained indicates user pods have been evicted. + ConditionPodsDrained HealthEventConditionType = "PodsDrained" + + // ConditionRemediated indicates remediation has completed. + ConditionRemediated HealthEventConditionType = "Remediated" + + // ConditionResolved indicates the event has been fully resolved. + ConditionResolved HealthEventConditionType = "Resolved" +) diff --git a/api/nvsentinel/v1alpha1/zz_generated.deepcopy.go b/api/nvsentinel/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000..3e5d82626 --- /dev/null +++ b/api/nvsentinel/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,267 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ActionOverride) DeepCopyInto(out *ActionOverride) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ActionOverride. +func (in *ActionOverride) DeepCopy() *ActionOverride { + if in == nil { + return nil + } + out := new(ActionOverride) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BehaviourOverrides) DeepCopyInto(out *BehaviourOverrides) { + *out = *in + if in.Quarantine != nil { + in, out := &in.Quarantine, &out.Quarantine + *out = new(ActionOverride) + **out = **in + } + if in.Drain != nil { + in, out := &in.Drain, &out.Drain + *out = new(ActionOverride) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BehaviourOverrides. +func (in *BehaviourOverrides) DeepCopy() *BehaviourOverrides { + if in == nil { + return nil + } + out := new(BehaviourOverrides) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Entity) DeepCopyInto(out *Entity) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Entity. +func (in *Entity) DeepCopy() *Entity { + if in == nil { + return nil + } + out := new(Entity) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HealthEvent) DeepCopyInto(out *HealthEvent) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HealthEvent. +func (in *HealthEvent) DeepCopy() *HealthEvent { + if in == nil { + return nil + } + out := new(HealthEvent) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *HealthEvent) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HealthEventCondition) DeepCopyInto(out *HealthEventCondition) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HealthEventCondition. +func (in *HealthEventCondition) DeepCopy() *HealthEventCondition { + if in == nil { + return nil + } + out := new(HealthEventCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HealthEventList) DeepCopyInto(out *HealthEventList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]HealthEvent, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HealthEventList. +func (in *HealthEventList) DeepCopy() *HealthEventList { + if in == nil { + return nil + } + out := new(HealthEventList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *HealthEventList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HealthEventSpec) DeepCopyInto(out *HealthEventSpec) { + *out = *in + if in.ErrorCodes != nil { + in, out := &in.ErrorCodes, &out.ErrorCodes + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.EntitiesImpacted != nil { + in, out := &in.EntitiesImpacted, &out.EntitiesImpacted + *out = make([]Entity, len(*in)) + copy(*out, *in) + } + in.DetectedAt.DeepCopyInto(&out.DetectedAt) + if in.Overrides != nil { + in, out := &in.Overrides, &out.Overrides + *out = new(BehaviourOverrides) + (*in).DeepCopyInto(*out) + } + if in.Metadata != nil { + in, out := &in.Metadata, &out.Metadata + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Maintenance != nil { + in, out := &in.Maintenance, &out.Maintenance + *out = new(MaintenanceInfo) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HealthEventSpec. +func (in *HealthEventSpec) DeepCopy() *HealthEventSpec { + if in == nil { + return nil + } + out := new(HealthEventSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HealthEventStatus) DeepCopyInto(out *HealthEventStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]HealthEventCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.LastRemediationTime != nil { + in, out := &in.LastRemediationTime, &out.LastRemediationTime + *out = (*in).DeepCopy() + } + if in.ResolvedAt != nil { + in, out := &in.ResolvedAt, &out.ResolvedAt + *out = (*in).DeepCopy() + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HealthEventStatus. +func (in *HealthEventStatus) DeepCopy() *HealthEventStatus { + if in == nil { + return nil + } + out := new(HealthEventStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MaintenanceInfo) DeepCopyInto(out *MaintenanceInfo) { + *out = *in + if in.ScheduledStartTime != nil { + in, out := &in.ScheduledStartTime, &out.ScheduledStartTime + *out = (*in).DeepCopy() + } + if in.ScheduledEndTime != nil { + in, out := &in.ScheduledEndTime, &out.ScheduledEndTime + *out = (*in).DeepCopy() + } + if in.ActualStartTime != nil { + in, out := &in.ActualStartTime, &out.ActualStartTime + *out = (*in).DeepCopy() + } + if in.ActualEndTime != nil { + in, out := &in.ActualEndTime, &out.ActualEndTime + *out = (*in).DeepCopy() + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MaintenanceInfo. +func (in *MaintenanceInfo) DeepCopy() *MaintenanceInfo { + if in == nil { + return nil + } + out := new(MaintenanceInfo) + in.DeepCopyInto(out) + return out +} diff --git a/api/proto/device/v1alpha1/gpu.proto b/api/proto/device/v1alpha1/gpu.proto index 2a4cf7f86..dbb5aa7a8 100644 --- a/api/proto/device/v1alpha1/gpu.proto +++ b/api/proto/device/v1alpha1/gpu.proto @@ -14,26 +14,87 @@ syntax = "proto3"; -package nvidia.nvsentinel.v1alpha1; +package nvidia.device.v1alpha1; option go_package = "github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1;v1alpha1"; +import "google/protobuf/empty.proto"; import "google/protobuf/timestamp.proto"; // ========================================== // Resource Definitions // ========================================== +// ObjectMeta is a subset of k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. +message ObjectMeta { + // name is the unique logical identifier of the resource. Must be unique within a namespace. + string name = 1; + + // resource_version represents the internal version of this object. + // + // Value must be treated as opaque by clients and passed unmodified back to the server. + // Populated by the system. + // Read-only. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + string resource_version = 2; + + // namespace defines the space within which each name must be unique. An empty namespace is + // equivalent to the "default" namespace, but "default" is the canonical representation. + // Not all objects are required to be scoped to a namespace - the value of this field for + // those objects will be empty. + string namespace = 3; +} + +// ListMeta is a subset of k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta. +message ListMeta { + // resource_version identifies the version of the list snapshot. + // Clients can use this version to establish a watch from a consistent point in time. + // + // Value must be treated as opaque by clients and passed unmodified back to the server. + // Populated by the system. + // Read-only. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + string resource_version = 1; +} + +// GetOptions is the standard query options to the standard gRPC get call. +message GetOptions { + // resource_version sets a constraint on what resource versions a request may be served from. + // + // Optional. Defaults to unset. + string resource_version = 1; + + // namespace sets a constraint on what namespaces a request may be served from. + // An empty namespace is treated as a request for all namespaces. + // + // Optional. Defaults to unset. + string namespace = 2; +} + +// ListOptions is the query options to a standard gRPC list call. It is a subset of k8s.io/apimachinery/pkg/apis/meta/v1.ListOptions. +message ListOptions { + // resource_version sets a constraint on what resource versions a request may be served from. + // + // Optional. Defaults to unset. + string resource_version = 1; + + // namespace sets a constraint on what namespaces a request may be served from. + // An empty namespace is treated as a request for all namespaces. + // + // Optional. Defaults to unset. + string namespace = 2; +} + // Gpu represents a single GPU resource. // // Its structure follows the Kubernetes Resource Model pattern (Spec/Status). +// +// The resource name (metadata.name) is typically the lowercased GPU UUID, +// but may take other forms. +// +// Example: "gpu-a1b2c3d4-e5f6-a7b8-c9d0-e1f2a3b4c5d6" message Gpu { - // name is the unique logical identifier of the GPU resource. - // - // This is typically the lowercased GPU UUID, but may take other forms. - // - // Example: "gpu-a1b2c3d4-e5f6-a7b8-c9d0-e1f2a3b4c5d6" - string name = 1; + ObjectMeta metadata = 1; // spec defines the identity and desired attributes of the GPU resource. GpuSpec spec = 2; @@ -41,12 +102,17 @@ message Gpu { // status contains the most recently observed state of the GPU resource. // This data may lag slightly behind the actual on-device state. GpuStatus status = 3; + + // Note: resource_version is in metadata.resource_version, not as a top-level field. + // Use metadata.resource_version for optimistic concurrency control. } // GpuList is a collection of GPU resources. message GpuList { + ListMeta metadata = 1; + // items is the list of GPU resources. - repeated Gpu items = 1; + repeated Gpu items = 2; } // GpuSpec describes the identity and desired attributes of the GPU. @@ -92,16 +158,117 @@ message Condition { // Service Definition // ========================================== -// GpuService provides an API for observing GPU resources. +// GpuService provides a unified API for managing GPU resources. +// +// Read operations (Get, List, Watch) are intended for consumers like +// device plugins and DRA drivers. +// +// Write operations (Create, Update, UpdateStatus, Delete) are intended +// for providers like health monitors. +// +// All write operations acquire exclusive write locks, blocking ALL consumer +// read operations until the write completes. This ensures consumers never +// read stale data during status transitions. +// +// Access control should be enforced at the infrastructure level (e.g., +// NetworkPolicy, mTLS) rather than in the API itself. service GpuService { + // ========================================== + // Read Operations (Consumers + Providers) + // ========================================== + // GetGpu retrieves a single GPU resource by its unique name. + // + // Returns: + // - The GPU if found + // - NOT_FOUND if no GPU with that name exists rpc GetGpu(GetGpuRequest) returns (GetGpuResponse); - // ListGpus retrieves a list of GPU resources. + // ListGpus retrieves all GPU resources. + // + // Returns a GpuList containing all registered GPUs. rpc ListGpus(ListGpusRequest) returns (ListGpusResponse); // WatchGpus streams lifecycle events for GPU resources. + // + // Events are sent for: + // - ADDED: A new GPU was created + // - MODIFIED: An existing GPU was updated + // - DELETED: A GPU was removed + // + // The stream will receive an initial batch of ADDED events for all + // existing GPUs, followed by real-time events as changes occur. rpc WatchGpus(WatchGpusRequest) returns (stream WatchGpusResponse); + + // ========================================== + // Write Operations (Providers Only) + // ========================================== + + // CreateGpu registers a new GPU with the server. + // + // The GPU name (metadata.name) must be unique. This operation uses + // idempotent semantics: + // - If the GPU doesn't exist, it is created and created=true is returned + // - If a GPU with the same name exists, the existing GPU is returned + // with created=false (no error is returned) + // + // This enables safe retry patterns where providers can call CreateGpu + // without needing to handle ALREADY_EXISTS errors. + // + // This operation acquires a write lock, blocking consumer reads + // until the operation completes. + // + // Returns: + // - The GPU (created or existing) with server-assigned fields + // - INVALID_ARGUMENT if required fields are missing + rpc CreateGpu(CreateGpuRequest) returns (CreateGpuResponse); + + // UpdateGpu replaces an existing GPU resource. + // + // The entire GPU (spec and status) is replaced. Use UpdateGpuStatus + // if you only need to update the status. + // + // Optimistic concurrency: If resource_version is provided and does + // not match the current version, returns ABORTED. + // + // This operation acquires a write lock. + // + // Returns: + // - The updated GPU + // - NOT_FOUND if the GPU doesn't exist + // - ABORTED if resource_version doesn't match (conflict) + rpc UpdateGpu(UpdateGpuRequest) returns (UpdateGpuResponse); + + // UpdateGpuStatus updates only the status of an existing GPU. + // + // This is the primary method for health monitors to report GPU state. + // The spec is not modified. + // + // This follows the Kubernetes subresource pattern where status is + // updated separately from spec. + // + // This operation acquires a write lock, blocking consumer reads + // until the operation completes. This ensures consumers never read + // stale data during status transitions (e.g., healthy → unhealthy). + // + // Returns: + // - The updated GPU + // - NOT_FOUND if the GPU doesn't exist + // - ABORTED if resource_version doesn't match (conflict) + rpc UpdateGpuStatus(UpdateGpuStatusRequest) returns (UpdateGpuStatusResponse); + + // DeleteGpu removes a GPU from the server. + // + // After deletion, the GPU will no longer appear in ListGpus or + // GetGpu responses. Active WatchGpus streams will receive a + // DELETED event. + // + // This operation acquires a write lock. + // + // Returns: + // - Empty on success + // - NOT_FOUND if the GPU doesn't exist + rpc DeleteGpu(DeleteGpuRequest) returns (google.protobuf.Empty); } // ========================================== @@ -112,6 +279,9 @@ service GpuService { message GetGpuRequest { // The unique resource name of the GPU to retrieve. string name = 1; + + // opts contains the standard query and scoping options for the request. + GetOptions opts = 2; } // GetGpuResponse contains the requested GPU resource. @@ -121,10 +291,10 @@ message GetGpuResponse { } // ListGpusRequest specifies the criteria for listing GPU resources. -// -// NOTE: The request is currently empty, but reserved for future support -// of filtering and pagination. -message ListGpusRequest {} +message ListGpusRequest { + // opts contains the standard query and scoping options for the list. + ListOptions opts = 1; +} // ListGpusResponse contains the list of GPU resources. message ListGpusResponse { @@ -133,10 +303,10 @@ message ListGpusResponse { } // WatchGpusRequest specifies the parameters for the watch stream. -// -// NOTE: The request is currently empty, but reserved for future support -// of filtering and resource versioning (resumption). -message WatchGpusRequest {} +message WatchGpusRequest { + // opts contains the standard query and scoping options for the watch. + ListOptions opts = 1; +} // WatchGpusResponse describes a change event for a GPU resource. message WatchGpusResponse { @@ -155,3 +325,66 @@ message WatchGpusResponse { // For "ERROR", this may be empty or contain partial data. Gpu object = 2; } + +// CreateGpuRequest contains the GPU to create. +message CreateGpuRequest { + // The GPU to create. + // Required: metadata.name, spec.uuid + Gpu gpu = 1; +} + +// CreateGpuResponse contains the created GPU. +message CreateGpuResponse { + // gpu is the created GPU with server-assigned fields (resource_version). + Gpu gpu = 1; + + // created is true if a new GPU was created, false if it already existed. + // + // When false, the returned gpu contains the existing GPU data. + // This enables idempotent registration patterns. + bool created = 2; +} + +// UpdateGpuRequest contains the GPU to update. +message UpdateGpuRequest { + // The GPU to update. + // Required: metadata.name + // Optional: resource_version for optimistic concurrency + Gpu gpu = 1; +} + +// UpdateGpuResponse contains the updated GPU. +message UpdateGpuResponse { + // gpu is the updated GPU. + Gpu gpu = 1; +} + +// UpdateGpuStatusRequest contains the status update for a GPU. +message UpdateGpuStatusRequest { + // The name of the GPU to update. + // Required. + string name = 1; + + // The new status to set. + // This completely replaces the existing status. + // Required. + GpuStatus status = 2; + + // Expected resource_version for optimistic concurrency. + // If set and doesn't match current version, returns ABORTED. + // Optional. + string resource_version = 3; +} + +// UpdateGpuStatusResponse contains the updated GPU. +message UpdateGpuStatusResponse { + // gpu is the updated GPU. + Gpu gpu = 1; +} + +// DeleteGpuRequest specifies which GPU to delete. +message DeleteGpuRequest { + // The unique name of the GPU to delete. + // Required. + string name = 1; +} diff --git a/client-go/DEVELOPMENT.md b/client-go/DEVELOPMENT.md new file mode 100644 index 000000000..64f68110d --- /dev/null +++ b/client-go/DEVELOPMENT.md @@ -0,0 +1,57 @@ +# NVIDIA Device API: Go Client Development + +> [!IMPORTANT] +> This module relies heavily on **code generation**. For the high-level generation pipeline overview and the standard development loop, please refer to the [Root Development Guide](../DEVELOPMENT.md). + +--- + +## Internal Structure + +- `client/`: [Generated] The versioned Clientset. +- `listers/`: [Generated] Type-safe listers for cached lookups. +- `informers/`: [Generated] Shared Index Informers. +- `nvgrpc/`: **[Manual]** The gRPC transport layer +- `version/`: **[Manual]** Version injection functionality. + +--- + +## Local Workflow + +While global changes should be driven from the root Makefile, you can perform module-specific tasks here. + +### Building & Testing + +Unit tests in this directory focus on the manual logic in `nvgrpc` and the integrity of the generated clientset. + +```bash +# Verify type safety of generated code and manual logic +make build + +# Run unit tests +make test +``` + +### Modifying Generated Code + +> [!WARNING] +> **Do not edit generated files directly.** +> +> Files in `client/`, `listers/`, and `informers/` contain a `DO NOT EDIT` header and are overwritten every time you run `make code-gen`. +> +> To change behavior: +> - **API Definitions**: Modify the Proto or Go types in `../api`. +> - **Client Logic**: Modify the templates in `../code-generator/cmd/client-gen`. +> - **Transport & Connection**: Modify the gRPC logic in `nvgrpc`. + +## Housekeeping + +If your generated files are out of sync or contain stale data, you can purge the local generated artifacts: + +```bash +# Removes generated code (client, listers, informers) +make clean +``` + +> [!TIP] For broader repository issues or environment-wise troubleshooting, please refer to the [Root Development Guide](../DEVELOPMENT.md). + +--- diff --git a/client-go/Makefile b/client-go/Makefile new file mode 100644 index 000000000..254f0693e --- /dev/null +++ b/client-go/Makefile @@ -0,0 +1,74 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ============================================================================== +# Configuration +# ============================================================================== + +MODULE_NAME = $(shell basename $(shell pwd)) + +# Versioning +GIT_VERSION ?= $(shell git describe --tags --always --dirty) +VERSION_PACKAGE := github.com/nvidia/nvsentinel/client-go/version +VERSION_FLAGS := -X '$(VERSION_PACKAGE).GitVersion=$(GIT_VERSION)' + +# Linker Flags +EXTRA_LDFLAGS ?= +LDFLAGS := -ldflags "$(VERSION_FLAGS) $(EXTRA_LDFLAGS)" + +# Shell setup +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + +# ============================================================================== +# Targets +# ============================================================================== + +.PHONY: all +all: code-gen test build ## Run code generation, compile all code, and execute tests. + +##@ General + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Development + +.PHONY: code-gen +code-gen: ## Run code generation (Client, Listers, Informers). + @echo "Generating code ($(MODULE_NAME))..." + ./hack/update-codegen.sh + @echo "Synchronizing module dependencies..." + go mod tidy + +.PHONY: build +build: ## Compile all Go code after generation to verify type safety. + @echo "Compiling ($(MODULE_NAME))..." + go build -v $(LDFLAGS) $$(go list ./... | grep -vE '/hack/overlays/|/examples/|/tests/') + +.PHONY: test +test: ## Run unit tests and generate coverage. + @echo "Testing ($(MODULE_NAME))..." + go test -v $$(go list ./... | grep -vE '/hack/overlays/|/client/|/listers/|informers/|/fake|/scheme|/typed/|/examples/') -coverprofile cover.out + +.PHONY: lint +lint: ## Run golangci-lint. + @echo "Linting ($(MODULE_NAME))..." + golangci-lint run ./... + +.PHONY: clean +clean: ## Remove all generated code. + @echo "Cleaning generated code ($(MODULE_NAME))..." + rm -rf client listers informers diff --git a/client-go/README.md b/client-go/README.md new file mode 100644 index 000000000..6de60f319 --- /dev/null +++ b/client-go/README.md @@ -0,0 +1,101 @@ +# NVIDIA Device API: Go Client + +The `client-go` module is the official Go SDK for interacting with the node-local NVIDIA Device API. It provides a Kubernetes-native developer experience for building node-level agents, telemetry sidecars, and operators **driven by local device state.** + +By utilizing a node-local gRPC transport, this SDK allows agents to query device telemetry and status **without putting load on the central Kubernetes API server**. This architecture enables fine-grained **hardware monitoring** that scales independently of the cluster control plane. + +> [!WARNING] +> **Experimental Preview Release** +> This is an experimental release of the NVIDIA Device API Go client. Use at your own risk in production environments. The software is provided "as is" without warranties of any kind. Features, APIs, and configurations may change without notice in future releases. For production deployments, thoroughly test in non-critical environments first. +> +> **Capabilities (Read-Only)** +> * ✅ **Supported**: `Get`, `List`, `Watch` +> * ❌ **Unsupported**: `Create`, `Update`, `UpdateStatus`, `Patch`, `Delete` + +## Features + +- **Kubernetes-Native API**: Provides generated versioned clientsets, informers, and listers that mirror standard K8s interfaces. +- **gRPC Transport**: Optimized for low-latency, local communication via Unix domain sockets (UDS). +- **controller-runtime Integration**: Supports **Informer Injection** to drive standard Reconcilers with local gRPC streams. +- **Observability**: Integrated Prometheus metrics, error logging, and support for structured logging. + +## Installation + +```bash +go get github.com/nvidia/nvsentinel/client-go +``` + +## Quick Start + +The following snippet demonstrates how to initialize the client and retrieve a list of GPUs from the local node. + +```go +package main + +import ( + "context" + "fmt" + "log" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/api/meta" + "github.com/nvidia/nvsentinel/client-go/client/versioned" + "github.com/nvidia/nvsentinel/client-go/nvgrpc" +) +func main() { + config := &nvgrpc.Config{Target: "unix:///var/run/nvidia-device-api/device-api.sock"} + clientset := versioned.NewForConfigOrDie(config) + + gpus, err := clientset.DeviceV1alpha1().GPUs().List(context.Background(), metav1.ListOptions{}) + if err != nil { + log.Fatalf("failed to list GPUs: %v", err) + } + + for _, gpu := range gpus.Items { + isReady := meta.IsStatusConditionTrue(gpu.Status.Conditions, "Ready") + fmt.Printf("GPU: %s | Ready: %v\n", gpu.Name, isReady) + } +} +``` + +## Integration Patterns + +| Pattern | Focus | Description | +| :--- | :--- | :--- | +| **[Basic Client](./examples/client)** | **Point-in-Time** | Initializing the clientset, listing resources, and inspecting status. | +| **[Watch Monitor](./examples/watch)** | **Event-Driven** | Using gRPC interceptors and asynchronous `Watch` streams. | +| **[Controller](./examples/controller)** | **State Enforcement** | Driving a `controller-runtime` reconciler with node-local gRPC data. | +| **[Fake Client](./examples/fake-client)** | **Unit Testing** | Testing logic using the in-memory `ObjectTracker`. | + +## Deployment + +Clients are typically deployed as a **DaemonSet**. The following configuration is required to ensure connectivity to the host's NVIDIA Device API socket. + +### Volume Mounts + +```yaml +volumeMounts: +- name: device-api-socket + mountPath: /var/run/nvidia-device-api + readOnly: true +volumes: +- name: device-api-socket + hostPath: + path: /var/run/nvidia-device-api + type: DirectoryOrCreate +``` + +### Configuration + +| Variable | Description | Default | +| :--- | :--- | :--- | +| **`NVIDIA_DEVICE_API_TARGET`** | gRPC target URI (e.g., `unix:///...`) | `unix:///var/run/nvidia-device-api/device-api.sock` | + +> [!NOTE] +> The container user must have filesystem permissions to read/write to the Unix socket file. + +## Development + +Refer to this module's [Development Guide](./DEVELOPMENT.md) for instructions on building the client and running the code generation pipeline. + +--- diff --git a/client-go/client/versioned/clientset.go b/client-go/client/versioned/clientset.go new file mode 100644 index 000000000..aeec40f73 --- /dev/null +++ b/client-go/client/versioned/clientset.go @@ -0,0 +1,105 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by client-gen. DO NOT EDIT. + +package versioned + +import ( + fmt "fmt" + + devicev1alpha1 "github.com/nvidia/nvsentinel/client-go/client/versioned/typed/device/v1alpha1" + nvgrpc "github.com/nvidia/nvsentinel/client-go/nvgrpc" + grpc "google.golang.org/grpc" +) + +type Interface interface { + DeviceV1alpha1() devicev1alpha1.DeviceV1alpha1Interface +} + +// Clientset contains the clients for groups. +type Clientset struct { + deviceV1alpha1 *devicev1alpha1.DeviceV1alpha1Client +} + +// DeviceV1alpha1 retrieves the DeviceV1alpha1Client +func (c *Clientset) DeviceV1alpha1() devicev1alpha1.DeviceV1alpha1Interface { + return c.deviceV1alpha1 +} + +// NewForConfig creates a new Clientset for the given config. +// NewForConfig is equivalent to NewForConfigAndClient(c, clientConn), +// where clientConn was generated with nvgrpc.ClientConnFor(c). +// +// If you need to customize the connection (e.g. set a logger), +// use nvgrpc.ClientConnFor() manually and pass the connection to NewForConfigAndClient. +func NewForConfig(c *nvgrpc.Config) (*Clientset, error) { + if c == nil { + return nil, fmt.Errorf("config cannot be nil") + } + + configShallowCopy := *c // Shallow copy to avoid mutation + conn, err := nvgrpc.ClientConnFor(&configShallowCopy) + if err != nil { + return nil, err + } + + cs, err := NewForConfigAndClient(&configShallowCopy, conn) + if err != nil { + // Close connection to prevent resource leak + _ = conn.Close() + return nil, err + } + return cs, nil +} + +// NewForConfigAndClient creates a new Clientset for the given config and gRPC client connection. +// The provided gRPC client connection provided takes precedence over the configured transport values. +func NewForConfigAndClient(c *nvgrpc.Config, conn grpc.ClientConnInterface) (*Clientset, error) { + if c == nil { + return nil, fmt.Errorf("config cannot be nil") + } + if conn == nil { + return nil, fmt.Errorf("gRPC connection cannot be nil") + } + + configShallowCopy := *c // Shallow copy to avoid mutation + + var cs Clientset + var err error + cs.deviceV1alpha1, err = devicev1alpha1.NewForConfigAndClient(&configShallowCopy, conn) + if err != nil { + return nil, err + } + + return &cs, nil +} + +// NewForConfigOrDie creates a new Clientset for the given config and +// panics if there is an error in the config or connection setup. +func NewForConfigOrDie(c *nvgrpc.Config) *Clientset { + cs, err := NewForConfig(c) + if err != nil { + panic(err) + } + return cs +} + +// New creates a new Clientset for the given gRPC client connection. +func New(conn grpc.ClientConnInterface) *Clientset { + var cs Clientset + cs.deviceV1alpha1 = devicev1alpha1.New(conn) + + return &cs +} diff --git a/client-go/client/versioned/fake/clientset_generated.go b/client-go/client/versioned/fake/clientset_generated.go new file mode 100644 index 000000000..c1e2b15cf --- /dev/null +++ b/client-go/client/versioned/fake/clientset_generated.go @@ -0,0 +1,105 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + clientset "github.com/nvidia/nvsentinel/client-go/client/versioned" + devicev1alpha1 "github.com/nvidia/nvsentinel/client-go/client/versioned/typed/device/v1alpha1" + fakedevicev1alpha1 "github.com/nvidia/nvsentinel/client-go/client/versioned/typed/device/v1alpha1/fake" + grpc "google.golang.org/grpc" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/discovery" + fakediscovery "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/testing" +) + +// NewSimpleClientset returns a clientset that will respond with the provided objects. +// It's backed by a very simple object tracker that processes creates, updates and deletions as-is, +// without applying any field management, validations and/or defaults. It shouldn't be considered a replacement +// for a real clientset and is mostly useful in simple unit tests. +func NewSimpleClientset(objects ...runtime.Object) *Clientset { + o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) + for _, obj := range objects { + if err := o.Add(obj); err != nil { + panic(err) + } + } + + cs := &Clientset{tracker: o} + cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} + cs.AddReactor("*", "*", testing.ObjectReaction(o)) + cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { + var opts metav1.ListOptions + if watchAction, ok := action.(testing.WatchActionImpl); ok { + opts = watchAction.ListOptions + } + gvr := action.GetResource() + ns := action.GetNamespace() + watch, err := o.Watch(gvr, ns, opts) + if err != nil { + return false, nil, err + } + return true, watch, nil + }) + + return cs +} + +// Clientset implements clientset.Interface. Meant to be embedded into a +// struct to get a default implementation. This makes faking out just the method +// you want to test easier. +type Clientset struct { + testing.Fake + discovery *fakediscovery.FakeDiscovery + tracker testing.ObjectTracker +} + +// ClientConn returns nil for fake clientsets. There is no real gRPC connection. +func (c *Clientset) ClientConn() grpc.ClientConnInterface { + return nil +} + +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + return c.discovery +} + +func (c *Clientset) Tracker() testing.ObjectTracker { + return c.tracker +} + +// IsWatchListSemanticsUnSupported informs the reflector that this client +// doesn't support WatchList semantics. +// +// This is a synthetic method whose sole purpose is to satisfy the optional +// interface check performed by the reflector. +// Returning true signals that WatchList can NOT be used. +// No additional logic is implemented here. +func (c *Clientset) IsWatchListSemanticsUnSupported() bool { + return true +} + +var ( + _ clientset.Interface = &Clientset{} + _ testing.FakeClient = &Clientset{} +) + +// DeviceV1alpha1 retrieves the DeviceV1alpha1Client +func (c *Clientset) DeviceV1alpha1() devicev1alpha1.DeviceV1alpha1Interface { + return &fakedevicev1alpha1.FakeDeviceV1alpha1{Fake: &c.Fake} +} diff --git a/client-go/client/versioned/fake/doc.go b/client-go/client/versioned/fake/doc.go new file mode 100644 index 000000000..44b048c89 --- /dev/null +++ b/client-go/client/versioned/fake/doc.go @@ -0,0 +1,18 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated fake clientset. +package fake diff --git a/client-go/client/versioned/fake/register.go b/client-go/client/versioned/fake/register.go new file mode 100644 index 000000000..1573cb4f7 --- /dev/null +++ b/client-go/client/versioned/fake/register.go @@ -0,0 +1,54 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + devicev1alpha1 "github.com/nvidia/nvsentinel/api/device/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +var scheme = runtime.NewScheme() +var codecs = serializer.NewCodecFactory(scheme) + +var localSchemeBuilder = runtime.SchemeBuilder{ + devicev1alpha1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(scheme)) +} diff --git a/client-go/client/versioned/scheme/doc.go b/client-go/client/versioned/scheme/doc.go new file mode 100644 index 000000000..55f52dc51 --- /dev/null +++ b/client-go/client/versioned/scheme/doc.go @@ -0,0 +1,18 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by client-gen. DO NOT EDIT. + +// This package contains the scheme of the automatically generated clientset. +package scheme diff --git a/client-go/client/versioned/scheme/register.go b/client-go/client/versioned/scheme/register.go new file mode 100644 index 000000000..97cf5a8ff --- /dev/null +++ b/client-go/client/versioned/scheme/register.go @@ -0,0 +1,54 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by client-gen. DO NOT EDIT. + +package scheme + +import ( + devicev1alpha1 "github.com/nvidia/nvsentinel/api/device/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +var Scheme = runtime.NewScheme() +var Codecs = serializer.NewCodecFactory(Scheme) +var ParameterCodec = runtime.NewParameterCodec(Scheme) +var localSchemeBuilder = runtime.SchemeBuilder{ + devicev1alpha1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(Scheme)) +} diff --git a/client-go/client/versioned/typed/device/v1alpha1/device_client.go b/client-go/client/versioned/typed/device/v1alpha1/device_client.go new file mode 100644 index 000000000..74aeaf588 --- /dev/null +++ b/client-go/client/versioned/typed/device/v1alpha1/device_client.go @@ -0,0 +1,100 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + fmt "fmt" + + logr "github.com/go-logr/logr" + nvgrpc "github.com/nvidia/nvsentinel/client-go/nvgrpc" + grpc "google.golang.org/grpc" +) + +type DeviceV1alpha1Interface interface { + ClientConn() grpc.ClientConnInterface + GPUsGetter +} + +// DeviceV1alpha1Client is used to interact with features provided by the device.nvidia.com group. +type DeviceV1alpha1Client struct { + conn grpc.ClientConnInterface + logger logr.Logger +} + +func (c *DeviceV1alpha1Client) GPUs() GPUInterface { + return newGPUs(c) +} + +// NewForConfig creates a new DeviceV1alpha1Client for the given config. +// NewForConfig is equivalent to NewForConfigAndClient(c, clientConn), +// where clientConn was generated with nvgrpc.ClientConnFor(c). +func NewForConfig(c *nvgrpc.Config) (*DeviceV1alpha1Client, error) { + if c == nil { + return nil, fmt.Errorf("config cannot be nil") + } + + config := *c // Shallow copy to avoid mutation + conn, err := nvgrpc.ClientConnFor(&config) + if err != nil { + return nil, err + } + + return NewForConfigAndClient(&config, conn) +} + +// NewForConfigAndClient creates a new DeviceV1alpha1Client for the given config and gRPC client connection. +// Note the grpc client connection provided takes precedence over the configured transport values. +func NewForConfigAndClient(c *nvgrpc.Config, conn grpc.ClientConnInterface) (*DeviceV1alpha1Client, error) { + if c == nil { + return nil, fmt.Errorf("config cannot be nil") + } + if conn == nil { + return nil, fmt.Errorf("gRPC connection cannot be nil") + } + + return &DeviceV1alpha1Client{ + conn: conn, + logger: c.GetLogger().WithName("device.nvidia.com.v1alpha1"), + }, nil +} + +// NewForConfigOrDie creates a new DeviceV1alpha1Client for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *nvgrpc.Config) *DeviceV1alpha1Client { + client, err := NewForConfig(c) + if err != nil { + panic(err) + } + return client +} + +// New creates a new DeviceV1alpha1Client for the given gRPC client connection. +func New(c grpc.ClientConnInterface) *DeviceV1alpha1Client { + return &DeviceV1alpha1Client{ + conn: c, + logger: logr.Discard(), + } +} + +// ClientConn returns a gRPC client connection that is used to communicate +// with API server by this client implementation. +func (c *DeviceV1alpha1Client) ClientConn() grpc.ClientConnInterface { + if c == nil { + return nil + } + return c.conn +} diff --git a/client-go/client/versioned/typed/device/v1alpha1/doc.go b/client-go/client/versioned/typed/device/v1alpha1/doc.go new file mode 100644 index 000000000..7749c1800 --- /dev/null +++ b/client-go/client/versioned/typed/device/v1alpha1/doc.go @@ -0,0 +1,18 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated typed clients. +package v1alpha1 diff --git a/client-go/client/versioned/typed/device/v1alpha1/fake/doc.go b/client-go/client/versioned/typed/device/v1alpha1/fake/doc.go new file mode 100644 index 000000000..2702a5453 --- /dev/null +++ b/client-go/client/versioned/typed/device/v1alpha1/fake/doc.go @@ -0,0 +1,18 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by client-gen. DO NOT EDIT. + +// Package fake has the automatically generated clients. +package fake diff --git a/client-go/client/versioned/typed/device/v1alpha1/fake/fake_device_client.go b/client-go/client/versioned/typed/device/v1alpha1/fake/fake_device_client.go new file mode 100644 index 000000000..f8548bb5f --- /dev/null +++ b/client-go/client/versioned/typed/device/v1alpha1/fake/fake_device_client.go @@ -0,0 +1,36 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1alpha1 "github.com/nvidia/nvsentinel/client-go/client/versioned/typed/device/v1alpha1" + grpc "google.golang.org/grpc" + testing "k8s.io/client-go/testing" +) + +type FakeDeviceV1alpha1 struct { + *testing.Fake +} + +// ClientConn returns nil for fake clientsets. There is no real gRPC connection. +func (c *FakeDeviceV1alpha1) ClientConn() grpc.ClientConnInterface { + return nil +} + +func (c *FakeDeviceV1alpha1) GPUs() v1alpha1.GPUInterface { + return newFakeGPUs(c) +} diff --git a/client-go/client/versioned/typed/device/v1alpha1/fake/fake_gpu.go b/client-go/client/versioned/typed/device/v1alpha1/fake/fake_gpu.go new file mode 100644 index 000000000..06147a477 --- /dev/null +++ b/client-go/client/versioned/typed/device/v1alpha1/fake/fake_gpu.go @@ -0,0 +1,89 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + context "context" + + devicev1alpha1 "github.com/nvidia/nvsentinel/api/device/v1alpha1" + v1alpha1 "github.com/nvidia/nvsentinel/client-go/client/versioned/typed/device/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + schema "k8s.io/apimachinery/pkg/runtime/schema" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// GPUsGetter has a method to return a GPUInterface. +// A group's client should implement this interface. +type GPUsGetter interface { + GPUs() v1alpha1.GPUInterface +} + +// fakeGPUs implements GPUInterface +type fakeGPUs struct { + Fake *FakeDeviceV1alpha1 +} + +// newFakeGPUs returns a fakeGPUs +func newFakeGPUs(fake *FakeDeviceV1alpha1) v1alpha1.GPUInterface { + return &fakeGPUs{ + Fake: fake, + } +} + +func (c *fakeGPUs) Resource() schema.GroupVersionResource { + return devicev1alpha1.SchemeGroupVersion.WithResource("gpus") +} + +func (c *fakeGPUs) Kind() schema.GroupVersionKind { + return devicev1alpha1.SchemeGroupVersion.WithKind("GPU") +} + +func (c *fakeGPUs) GetNamespace() string { + if c == nil { + return "" + } + return "" +} + +// Get takes name of the GPU, and returns the corresponding GPU object, and an error if there is any. +func (c *fakeGPUs) Get(ctx context.Context, name string, opts v1.GetOptions) (result *devicev1alpha1.GPU, err error) { + emptyResult := &devicev1alpha1.GPU{} + obj, err := c.Fake. + Invokes(testing.NewRootGetActionWithOptions(c.Resource(), name, opts), emptyResult) + if obj == nil { + return emptyResult, err + } + return obj.(*devicev1alpha1.GPU), err +} + +// List takes label and field selectors, and returns the list of GPUs that match those selectors. +func (c *fakeGPUs) List(ctx context.Context, opts v1.ListOptions) (result *devicev1alpha1.GPUList, err error) { + emptyResult := &devicev1alpha1.GPUList{} + obj, err := c.Fake. + Invokes(testing.NewRootListActionWithOptions(c.Resource(), c.Kind(), opts), emptyResult) + if obj == nil { + return emptyResult, err + } + return obj.(*devicev1alpha1.GPUList), err +} + +// Watch returns a watch.Interface that watches the requested GPUs. +func (c *fakeGPUs) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewRootWatchActionWithOptions(devicev1alpha1.SchemeGroupVersion.WithResource("gpus"), opts)) +} diff --git a/client-go/client/versioned/typed/device/v1alpha1/generated_expansion.go b/client-go/client/versioned/typed/device/v1alpha1/generated_expansion.go new file mode 100644 index 000000000..c99bbb48c --- /dev/null +++ b/client-go/client/versioned/typed/device/v1alpha1/generated_expansion.go @@ -0,0 +1,19 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +type GPUExpansion interface{} diff --git a/client-go/client/versioned/typed/device/v1alpha1/gpu.go b/client-go/client/versioned/typed/device/v1alpha1/gpu.go new file mode 100644 index 000000000..665fac751 --- /dev/null +++ b/client-go/client/versioned/typed/device/v1alpha1/gpu.go @@ -0,0 +1,156 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + context "context" + "fmt" + + logr "github.com/go-logr/logr" + devicev1alpha1 "github.com/nvidia/nvsentinel/api/device/v1alpha1" + pb "github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1" + nvgrpc "github.com/nvidia/nvsentinel/client-go/nvgrpc" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" +) + +// GPUsGetter has a method to return a GPUInterface. +// A group's client should implement this interface. +type GPUsGetter interface { + GPUs() GPUInterface +} + +// GPUInterface has methods to work with GPU resources. +type GPUInterface interface { + Get(ctx context.Context, name string, opts v1.GetOptions) (*devicev1alpha1.GPU, error) + List(ctx context.Context, opts v1.ListOptions) (*devicev1alpha1.GPUList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + GPUExpansion +} + +// gpus implements GPUInterface +type gpus struct { + client pb.GpuServiceClient + logger logr.Logger +} + +// newGPUs returns a gpus +func newGPUs(c *DeviceV1alpha1Client) *gpus { + return &gpus{ + client: pb.NewGpuServiceClient(c.ClientConn()), + logger: c.logger.WithName("gpus"), + } +} + +func (c *gpus) getNamespace() string { + if c == nil { + return "" + } + return "" +} + +func (c *gpus) Get(ctx context.Context, name string, opts v1.GetOptions) (*devicev1alpha1.GPU, error) { + resp, err := c.client.GetGpu(ctx, &pb.GetGpuRequest{ + Name: name, + Opts: &pb.GetOptions{ + ResourceVersion: opts.ResourceVersion, + Namespace: c.getNamespace(), + }, + }) + if err != nil { + return nil, err + } + + obj := devicev1alpha1.FromProto(resp.GetGpu()) + if obj == nil { + return nil, fmt.Errorf("received nil GPU from server for name %s", name) + } + c.logger.V(6).Info("Fetched GPU", + "name", name, + "namespace", c.getNamespace(), + "resource-version", obj.GetResourceVersion(), + ) + + return obj, nil +} + +func (c *gpus) List(ctx context.Context, opts v1.ListOptions) (*devicev1alpha1.GPUList, error) { + resp, err := c.client.ListGpus(ctx, &pb.ListGpusRequest{ + Opts: &pb.ListOptions{ + ResourceVersion: opts.ResourceVersion, + Namespace: c.getNamespace(), + }, + }) + if err != nil { + return nil, err + } + + list := devicev1alpha1.FromProtoList(resp.GetGpuList()) + if list == nil { + return nil, fmt.Errorf("received nil GPU list from server for namespace %s", c.getNamespace()) + } + c.logger.V(5).Info("Listed GPUs", + "namespace", c.getNamespace(), + "count", len(list.Items), + "resource-version", list.GetResourceVersion(), + ) + + return list, nil +} + +func (c *gpus) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + c.logger.V(4).Info("Opening watch stream", + "resource", "gpus", + "namespace", c.getNamespace(), + "resource-version", opts.ResourceVersion, + ) + + ctx, cancel := context.WithCancel(ctx) + stream, err := c.client.WatchGpus(ctx, &pb.WatchGpusRequest{ + Opts: &pb.ListOptions{ + ResourceVersion: opts.ResourceVersion, + Namespace: c.getNamespace(), + }, + }) + if err != nil { + cancel() + return nil, err + } + + return nvgrpc.NewWatcher(&gpusStreamAdapter{stream: stream}, cancel, c.logger), nil +} + +// gpusStreamAdapter wraps the GPU gRPC stream to provide events. +type gpusStreamAdapter struct { + stream pb.GpuService_WatchGpusClient +} + +func (a *gpusStreamAdapter) Next() (string, runtime.Object, error) { + resp, err := a.stream.Recv() + if err != nil { + return "", nil, err + } + + obj := devicev1alpha1.FromProto(resp.GetObject()) + + return resp.GetType(), obj, nil +} + +func (a *gpusStreamAdapter) Close() error { + return a.stream.CloseSend() +} diff --git a/client-go/examples/README.md b/client-go/examples/README.md new file mode 100644 index 000000000..e069d811a --- /dev/null +++ b/client-go/examples/README.md @@ -0,0 +1,40 @@ +# NVIDIA Device API: Go Client Examples + +This directory contains examples demonstrating various use cases and functionality of the NVIDIA Device API Go client. + +--- + +## Integration Patterns + +| Example | Focus | Use Case | +| :--- | :--- | :--- | +| **[Basic Client](./client)** | **Point-in-Time Discovery** | CLI tools and scripts | +| **[Watch Monitor](./watch)** | **Asynchronous Operations** | Real-time monitoring and telemetry | +| **[Controller](./controller)** | **State Enforcement** | Kubernetes Operators and automation | +| **[Fake Client](./fake-client)** | **Unit Testing** | Mocking and event-driven validation | + +--- + +## Prerequisites + +All examples are designed to run locally using the [Fake Device API Server](./fake-server). It maintains an in-memory inventory of 8 GPUs and generates random status events (e.g., `Ready` toggles). + +```bash +# Start the simulated device api server +sudo go run ./fake-server/main.go +``` +To stop the server, press `Ctrl+C` + +## Running Examples + +With the server running in a dedicated terminal, navigate to any example and run it: + +```bash +# Running the basic client example +cd client/ +sudo go run main.go +``` + +> [!Note] `sudo` is required to access the default Unix domain socket (UDS) path in `/var/run/`. To run without root, override the socket path using the `NVIDIA_DEVICE_API_TARGET` environment variable. + +--- diff --git a/client-go/examples/client/README.md b/client-go/examples/client/README.md new file mode 100644 index 000000000..5d280510c --- /dev/null +++ b/client-go/examples/client/README.md @@ -0,0 +1,36 @@ +# NVIDIA Device API: Client Example + +This example demonstrates how to use the NVIDIA Device API client to interact with node-local resources. + +## Concepts + +* **Client Initialization**: Setting up a gRPC connection over a Unix domain socket (UDS) to communicate with the local device API. +* **K8s-Native Verbs**: Using standard `Get` and `List` operations to retrieve point-in-time resource snapshots. +* **Status Evaluation**: Using standard Kubernetes `meta` helpers to check resource readiness and conditions. + +--- + +## Running + +1. Ensure the [Fake Server](../fake-server) is running. +2. Run the example: + +```bash +sudo go run main.go +``` + +> [!NOTE] `sudo` is required for the default socket path in `/var/run/`. Override the path with the `NVIDIA_DEVICE_API_TARGET` environment variable if using a custom location. + +### Expected Output + +```text +"level"=0 "msg"="discovered GPUs" "count"=8 "target"="unix:///tmp/nvidia-device-api.sock" +"level"=0 "msg"="details" "name"="gpu-0" "uuid"="GPU-6e5b6a57..." +"level"=0 "msg"="status" "name"="gpu-0" "uuid"="GPU-6e5b6a57..." "status"="Ready" +"level"=0 "msg"="status" "name"="gpu-1" "uuid"="GPU-2b418863..." "status"="Ready" +"level"=0 "msg"="status" "name"="gpu-2" "uuid"="GPU-4e4e629e..." "status"="Ready" +... +"level"=0 "msg"="status" "name"="gpu-7" "uuid"="GPU-66ba2ccd..." "status"="NotReady" +``` + +--- diff --git a/client-go/examples/client/main.go b/client-go/examples/client/main.go new file mode 100644 index 000000000..2cc40e55f --- /dev/null +++ b/client-go/examples/client/main.go @@ -0,0 +1,93 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package main demonstrates the basic usage of the NVIDIA Device API client. +// +// It connects to the device-api server, lists all available GPUs, inspects +// their status fields using standard Kubernetes meta helpers, and logs the +// results to stdout. +package main + +import ( + "context" + "log" + "os" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/go-logr/stdr" + "github.com/nvidia/nvsentinel/client-go/client/versioned" + "github.com/nvidia/nvsentinel/client-go/nvgrpc" +) + +func main() { + logger := stdr.New(log.New(os.Stdout, "", log.LstdFlags)) + + // Determine the connection target. + // If the environment variable NVIDIA_DEVICE_API_TARGET is not set, use the + // default socket path: unix:///var/run/nvidia-device-api/device-api.sock + target := os.Getenv(nvgrpc.NvidiaDeviceAPITargetEnvVar) + if target == "" { + target = nvgrpc.DefaultNvidiaDeviceAPISocket + } + + // Initialize the versioned clientset using the gRPC transport. + config := &nvgrpc.Config{Target: target} + + clientset, err := versioned.NewForConfig(config) + if err != nil { + logger.Error(err, "unable to create clientset") + os.Exit(1) + } + + // List all GPUs to discover what is available on the node. + gpus, err := clientset.DeviceV1alpha1().GPUs().List(context.Background(), metav1.ListOptions{}) + if err != nil { + logger.Error(err, "failed to list GPUs") + os.Exit(1) + } + + logger.Info("discovered GPUs", "count", len(gpus.Items), "target", target) + + // Fetch a specific GPU by name. + if len(gpus.Items) > 0 { + firstName := gpus.Items[0].Name + + gpu, err := clientset.DeviceV1alpha1().GPUs().Get(context.Background(), firstName, metav1.GetOptions{}) + if err != nil { + logger.Error(err, "failed to fetch GPU", "name", firstName) + os.Exit(1) + } + + logger.Info("details", "name", gpu.Name, "uuid", gpu.Spec.UUID) + } + + // Inspect status conditions. + for _, gpu := range gpus.Items { + // Use standard K8s meta helpers to check status conditions safely. + isReady := meta.IsStatusConditionTrue(gpu.Status.Conditions, "Ready") + + status := "NotReady" + if isReady { + status = "Ready" + } + + logger.Info("status", + "name", gpu.Name, + "uuid", gpu.Spec.UUID, + "status", status, + ) + } +} diff --git a/client-go/examples/controller/README.md b/client-go/examples/controller/README.md new file mode 100644 index 000000000..ad0ee31f4 --- /dev/null +++ b/client-go/examples/controller/README.md @@ -0,0 +1,39 @@ +# NVIDIA Device API: Controller Example + +This example demonstrates how to use the NVIDIA Device API client with `controller-runtime` to drive standard Reconcilers. + +## Concepts + +- **Informer Injection**: Overriding `cache.Options` to inject a gRPC-backed `SharedIndexInformer` for specific Device types. +- **Hybrid Connectivity**: Setting up a `Manager` that maintains both a REST client (K8s API) and a gRPC client (Device API). +- **Transparent Caching**: Ensuring `mgr.GetClient()` calls are automatically routed to the local gRPC-backed cache for Device resources. +- **Controller-Runtime Integration**: Using standard `builder` patterns to set up controllers that react to local hardware events as if they were standard Kubernetes objects. + +--- + +## Running + +1. Ensure the [Fake Server](../fake-server) is running. +2. Run the controller: + +```bash +sudo go run main.go +``` +To stop the controller, press `Ctrl+C` + +> [!NOTE] `sudo` is required for the default socket path in `/var/run/`. Override the path with the `NVIDIA_DEVICE_API_TARGET` environment variable if using a custom location. + +### Expected Output + +```text +INFO setup starting manager +INFO controller-runtime.metrics Starting metrics server +INFO Starting EventSource {"controller": "gpu", "source": "kind source: *v1alpha1.GPU"} +INFO Starting Controller {"controller": "gpu"} +INFO Starting workers {"controller": "gpu", "worker count": 1} +INFO Reconciled GPU {"controller": "gpu", "name": "gpu-1", "uuid": "GPU-6bc0eaf0..."} +INFO Reconciled GPU {"controller": "gpu", "name": "gpu-2", "uuid": "GPU-0851fc7c..."} +... +``` + +--- diff --git a/client-go/examples/controller/main.go b/client-go/examples/controller/main.go new file mode 100644 index 000000000..a2d6c397e --- /dev/null +++ b/client-go/examples/controller/main.go @@ -0,0 +1,163 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package main demonstrates how to build a Kubernetes Controller for local devices. +// +// It uses controller-runtime to reconcile GPU resources, injecting a custom +// gRPC-backed Informer to bypass the standard Kubernetes API server and +// read directly from the local NVIDIA Device API socket. +package main + +import ( + "context" + "net/http" + "os" + "time" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/rest" + toolscache "k8s.io/client-go/tools/cache" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + devicev1alpha1 "github.com/nvidia/nvsentinel/api/device/v1alpha1" + "github.com/nvidia/nvsentinel/client-go/client/versioned" + "github.com/nvidia/nvsentinel/client-go/client/versioned/scheme" + informers "github.com/nvidia/nvsentinel/client-go/informers/externalversions" + "github.com/nvidia/nvsentinel/client-go/nvgrpc" +) + +func main() { + ctrl.SetLogger(zap.New(zap.UseDevMode(true))) + + setupLog := ctrl.Log.WithName("setup") + + // Determine the connection target. + // If the environment variable NVIDIA_DEVICE_API_TARGET is not set, use the + // default socket path: unix:///var/run/nvidia-device-api/device-api.sock + target := os.Getenv(nvgrpc.NvidiaDeviceAPITargetEnvVar) + if target == "" { + target = nvgrpc.DefaultNvidiaDeviceAPISocket + } + + // Initialize the versioned Clientset using the gRPC transport. + config := &nvgrpc.Config{Target: target} + + clientset, err := versioned.NewForConfig(config) + if err != nil { + setupLog.Error(err, "unable to create clientset") + os.Exit(1) + } + + // Create a factory for the gRPC-backed informers. + // We use a 10-minute resync period to ensure cache consistency. + // Note: We do not start the factory here; the Manager will start the injected informer. + factory := informers.NewSharedInformerFactory(clientset, 10*time.Minute) + gpuInformer := factory.Device().V1alpha1().GPUs().Informer() + + // Initialize the controller-runtime Manager. + // A dummy rest.Config is used here as we are not connecting to a real K8s API server. + mgr, err := ctrl.NewManager(&rest.Config{Host: "http://localhost:0"}, ctrl.Options{ + Scheme: scheme.Scheme, + // MapperProvider returns a RESTMapper that defines GPU resources as root-scoped. + // Required because the gRPC endpoint does not provide discovery APIs. + MapperProvider: func(c *rest.Config, httpClient *http.Client) (meta.RESTMapper, error) { + mapper := meta.NewDefaultRESTMapper([]schema.GroupVersion{devicev1alpha1.SchemeGroupVersion}) + mapper.Add(devicev1alpha1.SchemeGroupVersion.WithKind("GPU"), meta.RESTScopeRoot) + + return mapper, nil + }, + Cache: cache.Options{ + // NewInformer allows injecting a specific informer for a GroupVersionKind. + // This bypasses the default API server ListerWatcher for GPU resources. + NewInformer: func( + lw toolscache.ListerWatcher, + obj runtime.Object, + resync time.Duration, + indexers toolscache.Indexers, + ) toolscache.SharedIndexInformer { + if _, ok := obj.(*devicev1alpha1.GPU); ok { + // Merge the indexers required by controller-runtime with those + // already present in the gRPC informer. Conflicting keys (e.g., "namespace") + // are skipped to prefer the existing implementation. + existingIndexers := gpuInformer.GetIndexer().GetIndexers() + for key, indexerFunc := range indexers { + if _, exists := existingIndexers[key]; !exists { + err := gpuInformer.AddIndexers(toolscache.Indexers{key: indexerFunc}) + if err != nil { + setupLog.Error(err, "failed to add indexer to informer", "key", key) + os.Exit(1) + } + } + } + + return gpuInformer + } + // Fallback: For all other types, return a standard informer. This allows the + // manager to still handle standard Kubernetes resources (like Pods or Events) + // using the default API server transport, enabling "hybrid" reconciliation. + return toolscache.NewSharedIndexInformer(lw, obj, resync, indexers) + }, + }, + }) + if err != nil { + setupLog.Error(err, "unable to create manager") + os.Exit(1) + } + + if err = (&GPUReconciler{ + Client: mgr.GetClient(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "gpu") + os.Exit(1) + } + + ctx := ctrl.SetupSignalHandler() + + setupLog.Info("starting manager") + + if err := mgr.Start(ctx); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} + +// GPUReconciler reconciles GPUs using the local gRPC cache. +type GPUReconciler struct { + client.Client +} + +func (r *GPUReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := ctrl.LoggerFrom(ctx) + + var gpu devicev1alpha1.GPU + // The Get call is transparently routed through the injected gRPC-backed informer. + if err := r.Get(ctx, req.NamespacedName, &gpu); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + log.Info("Reconciled GPU", "name", gpu.Name, "uuid", gpu.Spec.UUID) + + return ctrl.Result{}, nil +} + +func (r *GPUReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&devicev1alpha1.GPU{}). + Complete(r) +} diff --git a/client-go/examples/fake-client/README.md b/client-go/examples/fake-client/README.md new file mode 100644 index 000000000..5800ea36b --- /dev/null +++ b/client-go/examples/fake-client/README.md @@ -0,0 +1,18 @@ +# NVIDIA Device API: Fake Client Example + +This example demonstrates how to use the NVIDIA Device API generated **fake versioned client** with a `SharedInformerFactory` in tests. + +## Concepts + +* **Fake Clientset**: Implements the `versioned.Interface` via an in-memory `ObjectTracker` to simulate the gRPC server state for unit testing. +* **Watch Reactors**: Using `PrependWatchReactor` to synchronize the transition from `LIST` to `WATCH`, preventing race conditions during event injection. +* **Tracker Injection**: Directly modifying the `ObjectTracker` to simulate "server-side" events, such as a discovery agent reporting a new device. + +--- + +## Running +```bash +go test -v -run TestGPUInformerWithFakeClient +``` + +--- diff --git a/client-go/examples/fake-client/main_test.go b/client-go/examples/fake-client/main_test.go new file mode 100644 index 000000000..fe384e4eb --- /dev/null +++ b/client-go/examples/fake-client/main_test.go @@ -0,0 +1,125 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// TestGPUInformerWithFakeClient demonstrates how to integrate a fake versioned +// clientset with a SharedInformerFactory in tests. +package main_test + +import ( + "context" + "testing" + "time" + + devicev1alpha1 "github.com/nvidia/nvsentinel/api/device/v1alpha1" + "github.com/nvidia/nvsentinel/client-go/client/versioned/fake" + "github.com/nvidia/nvsentinel/client-go/informers/externalversions" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" + clienttesting "k8s.io/client-go/testing" + "k8s.io/client-go/tools/cache" +) + +func TestGPUInformerWithFakeClient(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Initialize the fake clientset. The fake client uses an in-memory + // ObjectTracker to simulate the behavior of a real API server. + client := fake.NewSimpleClientset() + + // watcherStarted is used to synchronize the informer's transition from the + // initial LIST phase to the WATCH phase. + watcherStarted := make(chan struct{}) + + // Prepend a WatchReactor to intercept watch actions. This allows us to + // signal the test when the informer has successfully established its + // stream, preventing race conditions where events are injected before + // the watcher is ready. + client.PrependWatchReactor("*", func(action clienttesting.Action) (handled bool, ret watch.Interface, err error) { + watchAction, ok := action.(clienttesting.WatchActionImpl) + if !ok { + return false, nil, nil + } + + opts := watchAction.ListOptions + gvr := action.GetResource() + ns := action.GetNamespace() + + // Manually invoke the tracker to create the watch stream. + watch, err := client.Tracker().Watch(gvr, ns, opts) + if err != nil { + return false, nil, err + } + + // Close the channel to notify the test that the Informer is now + // listening for events. + close(watcherStarted) + return true, watch, nil + }) + + // Create a factory for the informers. + // We use a 0 resync period as we are testing event-driven logic. + gpuChan := make(chan *devicev1alpha1.GPU, 1) + factory := externalversions.NewSharedInformerFactory(client, 0) + gpuInformer := factory.Device().V1alpha1().GPUs().Informer() + + // Register an event handler to verify that the informer's cache is + // correctly updated and that notifications are dispatched. + gpuInformer.AddEventHandler(&cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + gpu := obj.(*devicev1alpha1.GPU) + t.Logf("Informer signaled GPU added: %s", gpu.Name) + gpuChan <- gpu + }, + }) + + // Start the informer factory and wait for the initial cache sync (LIST). + factory.Start(ctx.Done()) + if !cache.WaitForCacheSync(ctx.Done(), gpuInformer.HasSynced) { + t.Fatal("Timed out waiting for caches to sync") + } + + // Ensure the informer has moved past the LIST phase and into WATCH. + // In the fake client, writes that occur between LIST and WATCH are lost. + <-watcherStarted + + // Define a test resource to inject into the system. + testGPU := &devicev1alpha1.GPU{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gpu-1", + }, + Spec: devicev1alpha1.GPUSpec{ + UUID: "GPU-1", + }, + } + + // Inject the resource directly into the fake client's ObjectTracker. + // This simulates a "server-side" event, such as a discovery agent + // reporting a new device via the gRPC API. + err := client.Tracker().Add(testGPU) + if err != nil { + t.Fatalf("Tracker injection failed: %v", err) + } + + // Verify that the Informer successfully received and processed the ADD event. + select { + case gpu := <-gpuChan: + if gpu.Name != "gpu-1" { + t.Errorf("Expected GPU gpu-1, got %s", gpu.Name) + } + case <-time.After(wait.ForeverTestTimeout): + t.Error("Informer failed to receive the added GPU event within timeout") + } +} diff --git a/client-go/examples/fake-server/README.md b/client-go/examples/fake-server/README.md new file mode 100644 index 000000000..e6c772048 --- /dev/null +++ b/client-go/examples/fake-server/README.md @@ -0,0 +1,18 @@ +# NVIDIA Device API: Fake Device API Server + +This program simulates a running NVIDIA Device API server. It periodically modifies GPU readiness to provide a dynamic data source for testing without requiring a physical GPU node. + +--- + +## Usage + +Run the server in a dedicated terminal window: + +```bash +sudo go run main.go +``` +To stop the server, press `Ctrl+C` + +> [!NOTE] `sudo` is required for the default socket path in `/var/run/`. Override the path with `NVIDIA_DEVICE_API_TARGET` to run without root (e.g., `unix:///tmp/device-api.sock`). + +--- diff --git a/client-go/examples/fake-server/main.go b/client-go/examples/fake-server/main.go new file mode 100644 index 000000000..04c2a27d6 --- /dev/null +++ b/client-go/examples/fake-server/main.go @@ -0,0 +1,314 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package main implements a fake NVIDIA Device API server. +// +// It simulates a running device-api service over a Unix Domain Socket (UDS), +// maintaining an in-memory inventory of GPU resources and periodically +// toggling their readiness status to generate Watch events. +package main + +import ( + "context" + "crypto/rand" + "fmt" + "log" + "math/big" + "net" + "os" + "os/signal" + "path/filepath" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + devicev1alpha1 "github.com/nvidia/nvsentinel/api/device/v1alpha1" + pb "github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1" + "github.com/nvidia/nvsentinel/client-go/nvgrpc" +) + +func main() { + fmt.Printf("Starting server...\n") + + // Determine the connection target. + // If the environment variable NVIDIA_DEVICE_API_TARGET is not set, use the + // default socket path: unix:///var/run/nvidia-device-api/device-api.sock + target := os.Getenv(nvgrpc.NvidiaDeviceAPITargetEnvVar) + if target == "" { + target = nvgrpc.DefaultNvidiaDeviceAPISocket + } + + socketPath := strings.TrimPrefix(target, "unix://") + fmt.Printf("socketPath: %s\n", socketPath) + + if err := os.MkdirAll(filepath.Dir(socketPath), 0755); err != nil { + log.Fatalf("failed to create socket directory: %v", err) + } + + if _, err := os.Stat(socketPath); err == nil { + if err := os.Remove(socketPath); err != nil { + log.Fatalf("failed to remove stale socket: %v", err) + } + } + + lc := net.ListenConfig{} + + listener, err := lc.Listen(context.Background(), "unix", socketPath) + if err != nil { + log.Fatalf("failed to listen on %s: %v", socketPath, err) + } + + serverImpl := newFakeServer() + go serverImpl.simulateChanges() + + srv := grpc.NewServer() + pb.RegisterGpuServiceServer(srv, serverImpl) + + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + + go func() { + <-c + fmt.Println("\nStopping server...") + srv.GracefulStop() + os.Remove(socketPath) + os.Exit(0) + }() + + fmt.Printf("Fake Device API listening on %s\n", socketPath) + + if err := srv.Serve(listener); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} + +type fakeServer struct { + pb.UnimplementedGpuServiceServer + mu sync.RWMutex + gpus []devicev1alpha1.GPU + listeners map[chan struct{}]chan devicev1alpha1.GPU + currentRV int +} + +func newFakeServer() *fakeServer { + s := &fakeServer{ + gpus: make([]devicev1alpha1.GPU, 8), + listeners: make(map[chan struct{}]chan devicev1alpha1.GPU), + } + + for i := 0; i < 8; i++ { + s.gpus[i] = devicev1alpha1.GPU{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("gpu-%d", i), + ResourceVersion: "1", + }, + Spec: devicev1alpha1.GPUSpec{ + UUID: generateGPUUUID(), + }, + Status: devicev1alpha1.GPUStatus{ + Conditions: []metav1.Condition{ + { + Type: "Ready", + Status: metav1.ConditionTrue, + Reason: "DriverReady", + Message: "driver is posting ready status", + }, + }, + }, + } + } + + return s +} + +// simulateChanges flips the Ready status of a random GPU every few seconds +// to generate events for active Watch streams. +func (s *fakeServer) simulateChanges() { + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + for range ticker.C { + s.mu.Lock() + + // Pick a random GPU to update + idx := 0 + + if len(s.gpus) > 0 { + n, err := rand.Int(rand.Reader, big.NewInt(int64(len(s.gpus)))) + if err != nil { + idx = 0 + } else { + idx = int(n.Int64()) + } + } + + gpu := &s.gpus[idx] + + // Increment ResourceVersion for K8s watch semantics + s.currentRV++ + newRVStr := strconv.Itoa(s.currentRV) + gpu.ResourceVersion = newRVStr + + // Toggle the Ready condition + isReady := gpu.Status.Conditions[0].Status == metav1.ConditionTrue + + var newStatus metav1.ConditionStatus + if isReady { + newStatus = metav1.ConditionFalse + } else { + newStatus = metav1.ConditionTrue + } + + gpu.Status.Conditions[0].Status = newStatus + gpu.Status.Conditions[0].LastTransitionTime = metav1.Now() + + updatedGPU := *gpu.DeepCopy() + + s.mu.Unlock() + + s.broadcast(updatedGPU) + } +} + +func (s *fakeServer) broadcast(gpu devicev1alpha1.GPU) { + s.mu.RLock() + defer s.mu.RUnlock() + + for _, ch := range s.listeners { + select { + case ch <- gpu: + default: + } + } +} + +func (s *fakeServer) GetGpu(ctx context.Context, req *pb.GetGpuRequest) (*pb.GetGpuResponse, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + for _, gpu := range s.gpus { + if req.Name == gpu.Name { + return &pb.GetGpuResponse{Gpu: devicev1alpha1.ToProto(&gpu)}, nil + } + } + + return nil, status.Errorf(codes.NotFound, "gpu %q not found", req.Name) +} + +func (s *fakeServer) ListGpus(ctx context.Context, req *pb.ListGpusRequest) (*pb.ListGpusResponse, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + gpuList := &pb.GpuList{ + Items: make([]*pb.Gpu, 0, len(s.gpus)), + } + + for _, gpu := range s.gpus { + gpuList.Items = append(gpuList.Items, devicev1alpha1.ToProto(&gpu)) + } + + gpuList.Metadata = &pb.ListMeta{ + ResourceVersion: strconv.Itoa(s.currentRV), + } + + return &pb.ListGpusResponse{GpuList: gpuList}, nil +} + +//nolint:cyclop // This is an example; complexity is acceptable for clarity. +func (s *fakeServer) WatchGpus(req *pb.WatchGpusRequest, stream pb.GpuService_WatchGpusServer) error { + var requestRV int + if req.Opts.ResourceVersion != "" { + requestRV, _ = strconv.Atoi(req.Opts.ResourceVersion) + } + + if requestRV == 0 { + // Send Initial State (ADDED events) + var initial []devicev1alpha1.GPU + + s.mu.RLock() + + initial = make([]devicev1alpha1.GPU, len(s.gpus)) + for i, g := range s.gpus { + initial[i] = *g.DeepCopy() + } + + s.mu.RUnlock() + + for _, gpu := range initial { + if err := stream.Send(&pb.WatchGpusResponse{ + Type: "ADDED", + Object: devicev1alpha1.ToProto(&gpu), + }); err != nil { + return err + } + } + } + + // Register for updates + updateCh := make(chan devicev1alpha1.GPU, 10) + stopKey := make(chan struct{}) + + s.mu.Lock() + s.listeners[stopKey] = updateCh + s.mu.Unlock() + + defer func() { + s.mu.Lock() + delete(s.listeners, stopKey) + s.mu.Unlock() + }() + + log.Printf("Watch stream connected (starting RV: %d)", requestRV) + + // Stream updates + for { + select { + case <-stream.Context().Done(): + log.Println("Watch stream disconnected") + return nil + case gpu := <-updateCh: + gpuRV, _ := strconv.Atoi(gpu.ResourceVersion) + if gpuRV > requestRV { + err := stream.Send(&pb.WatchGpusResponse{ + Type: "MODIFIED", + Object: devicev1alpha1.ToProto(&gpu), + }) + if err != nil { + return err + } + } + } + } +} + +func generateGPUUUID() string { + b := make([]byte, 16) + + _, err := rand.Read(b) + if err != nil { + return "GPU-static-uuid-on-error" + } + + b[6] = (b[6] & 0x0f) | 0x40 + b[8] = (b[8] & 0x3f) | 0x80 + + return fmt.Sprintf("GPU-%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:]) +} diff --git a/client-go/examples/watch/README.md b/client-go/examples/watch/README.md new file mode 100644 index 000000000..fc1e2fc5d --- /dev/null +++ b/client-go/examples/watch/README.md @@ -0,0 +1,38 @@ +# NVIDIA Device API: Watch Example + +This example demonstrates how to set up and use the NVIDIA Device API client to react to asynchronous operations. + +## Concepts + +* **Manual Connection Management**: Constructing a connection with custom dialers for Unix domain sockets (UDS). +* **Middleware**: Injecting telemetry and structured logging into the gRPC transport layer. +* **Stream Processing**: Implementing long-lived `Watch()` streams with an asynchronous event-loop. +* **Context Handling**: Ensuring clean shutdowns by handling system signals (SIGTERM) and managing context-aware stream cancelation. + +--- + +## Running + +1. Ensure the [Fake Server](../fake-server) is running. +2. Run the example: + +```bash +sudo go run main.go +``` +To stop the application, press `Ctrl+C` + +> [!NOTE] `sudo` is required for the default socket path in `/var/run/`. Override the path with the `NVIDIA_DEVICE_API_TARGET` environment variable if using a custom location. + +### Expected Output + +```text +"level"=0 "msg"="retrieved GPU list" "count"=8 +"level"=0 "msg"="starting long-lived watch stream" "method"="/nvidia.nvsentinel.v1alpha1.GpuService/WatchGpus" +"level"=0 "msg"="watch stream established, waiting for events..." +"level"=0 "msg"="gpu status changed" "event"="ADDED" "name"="gpu-0" "uuid"="GPU-b56c1d18..." "status"="NotReady" +"level"=0 "msg"="gpu status changed" "event"="MODIFIED" "name"="gpu-0" "uuid"="GPU-b56c1d18..." "status"="Ready" +... +"level"=0 "msg"="gpu status changed" "event"="MODIFIED" "name"="gpu-1" "uuid"="GPU-2e6d5c15..." "status"="NotReady" +``` + +--- diff --git a/client-go/examples/watch/main.go b/client-go/examples/watch/main.go new file mode 100644 index 000000000..72428735e --- /dev/null +++ b/client-go/examples/watch/main.go @@ -0,0 +1,177 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package main demonstrates a long-running, event-driven agent. +// +// It establishes a persistent Watch stream with the device-api server, +// handling gRPC connection lifecycles, custom interceptors for telemetry, +// and real-time event processing for device state changes. +package main + +import ( + "context" + "fmt" + "log" + "os" + "os/signal" + "syscall" + + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + + "github.com/go-logr/stdr" + devicev1alpha1 "github.com/nvidia/nvsentinel/api/device/v1alpha1" + "github.com/nvidia/nvsentinel/client-go/client/versioned" + "github.com/nvidia/nvsentinel/client-go/nvgrpc" +) + +//nolint:cyclop,gocritic // This is an example; complexity and exitAfterDefer is acceptable for clarity. +func main() { + // Initialize a standard logger for transport-level visibility. + logger := stdr.New(log.New(os.Stdout, "", log.LstdFlags)) + + stdr.SetVerbosity(1) + + // Determine the connection target. + // If the environment variable NVIDIA_DEVICE_API_TARGET is not set, use the + // default socket path: unix:///var/run/nvidia-device-api/device-api.sock + target := os.Getenv(nvgrpc.NvidiaDeviceAPITargetEnvVar) + if target == "" { + target = nvgrpc.DefaultNvidiaDeviceAPISocket + } + + // tracingInterceptor injects metadata (x-request-id) into outgoing requests. + // This enables request tracking across the gRPC boundary. + tracingInterceptor := func( + ctx context.Context, + method string, + req, + reply any, + cc *grpc.ClientConn, + invoker grpc.UnaryInvoker, + opts ...grpc.CallOption, + ) error { + ctx = metadata.AppendToOutgoingContext(ctx, "x-request-id", "nv-trace-123") + return invoker(ctx, method, req, reply, cc, opts...) + } + + // watchMonitorInterceptor logs the start of long-lived Watch streams. + watchMonitorInterceptor := func( + ctx context.Context, + desc *grpc.StreamDesc, + cc *grpc.ClientConn, + method string, + streamer grpc.Streamer, + opts ...grpc.CallOption, + ) (grpc.ClientStream, error) { + logger.Info("starting long-lived watch stream", "method", method) + return streamer(ctx, desc, cc, method, opts...) + } + + // Configure manual DialOptions for transport-level control. + opts := []nvgrpc.DialOption{ + nvgrpc.WithLogger(logger), + nvgrpc.WithUnaryInterceptor(tracingInterceptor), + nvgrpc.WithStreamInterceptor(watchMonitorInterceptor), + } + + // Initialize the underlying gRPC connection manually. + config := &nvgrpc.Config{Target: target} + + conn, err := nvgrpc.ClientConnFor(config, opts...) + if err != nil { + logger.Error(err, "unable to connect to gRPC target") + os.Exit(1) + } + defer conn.Close() + + // Initialize the Clientset using the existing connection. + // This is required when specific gRPC lifecycle or interceptor management is needed. + clientset, err := versioned.NewForConfigAndClient(config, conn) + if err != nil { + logger.Error(err, "unable to create clientset") + os.Exit(1) + } + + // Create a context that is canceled when the app receives SIGINT or SIGTERM. + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + // List GPUs. This triggers the Unary interceptor. + list, err := clientset.DeviceV1alpha1().GPUs().List(ctx, metav1.ListOptions{}) + if err != nil { + logger.Error(err, "failed to list GPUs") + os.Exit(1) + } + + logger.Info("retrieved GPU list", "count", len(list.Items)) + + // Watch GPUs. This triggers the Stream interceptor. + watcher, err := clientset.DeviceV1alpha1().GPUs().Watch(ctx, metav1.ListOptions{ + ResourceVersion: list.ResourceVersion, + }) + if err != nil { + logger.Error(err, "failed to establish watch stream") + os.Exit(1) + } + + defer watcher.Stop() + + logger.Info("watch stream established, waiting for events...") + + for { + select { + case event, ok := <-watcher.ResultChan(): + if !ok { + logger.Info("watch channel closed by server") + return + } + + if event.Type == watch.Error { + if status, ok := event.Object.(*metav1.Status); ok { + logger.Info("received watch error from server", "reason", status.Reason, "message", status.Message) + } + + return + } + + gpu, ok := event.Object.(*devicev1alpha1.GPU) + if !ok { + logger.Info("received unknown object type", "type", fmt.Sprintf("%T", event.Object)) + continue + } + + isReady := meta.IsStatusConditionTrue(gpu.Status.Conditions, "Ready") + status := "NotReady" + + if isReady { + status = "Ready" + } + + logger.Info("gpu status changed", + "event", event.Type, + "name", gpu.Name, + "uuid", gpu.Spec.UUID, + "status", status, + ) + + case <-ctx.Done(): + logger.Info("received shutdown signal, stopping watch") + return + } + } +} diff --git a/client-go/go.mod b/client-go/go.mod new file mode 100644 index 000000000..c005830dc --- /dev/null +++ b/client-go/go.mod @@ -0,0 +1,76 @@ +module github.com/nvidia/nvsentinel/client-go + +go 1.25.5 + +replace github.com/nvidia/nvsentinel/api => ../api + +replace github.com/nvidia/nvsentinel/client-go => . + +require ( + github.com/go-logr/logr v1.4.3 + github.com/go-logr/stdr v1.2.2 + github.com/nvidia/nvsentinel/api v0.0.0 + google.golang.org/grpc v1.77.0 + k8s.io/apimachinery v0.35.0 + k8s.io/client-go v0.34.1 + sigs.k8s.io/controller-runtime v0.22.4 +) + +require ( + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/zapr v1.3.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.22.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/spf13/pflag v1.0.9 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/oauth2 v0.32.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/term v0.37.0 // indirect + golang.org/x/text v0.31.0 // indirect + golang.org/x/time v0.9.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251124214823-79d6a2a48846 // indirect + google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/api v0.34.1 // indirect + k8s.io/apiextensions-apiserver v0.34.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect +) diff --git a/client-go/go.sum b/client-go/go.sum new file mode 100644 index 000000000..02b16dcb5 --- /dev/null +++ b/client-go/go.sum @@ -0,0 +1,225 @@ +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= +github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= +github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= +github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= +github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= +golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251124214823-79d6a2a48846 h1:Wgl1rcDNThT+Zn47YyCXOXyX/COgMTIdhJ717F0l4xk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251124214823-79d6a2a48846/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM= +google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.34.1 h1:jC+153630BMdlFukegoEL8E/yT7aLyQkIVuwhmwDgJM= +k8s.io/api v0.34.1/go.mod h1:SB80FxFtXn5/gwzCoN6QCtPD7Vbu5w2n1S0J5gFfTYk= +k8s.io/apiextensions-apiserver v0.34.1 h1:NNPBva8FNAPt1iSVwIE0FsdrVriRXMsaWFMqJbII2CI= +k8s.io/apiextensions-apiserver v0.34.1/go.mod h1:hP9Rld3zF5Ay2Of3BeEpLAToP+l4s5UlxiHfqRaRcMc= +k8s.io/apimachinery v0.35.0 h1:Z2L3IHvPVv/MJ7xRxHEtk6GoJElaAqDCCU0S6ncYok8= +k8s.io/apimachinery v0.35.0/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= +k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY= +k8s.io/client-go v0.34.1/go.mod h1:kA8v0FP+tk6sZA0yKLRG67LWjqufAoSHA2xVGKw9Of8= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.22.4 h1:GEjV7KV3TY8e+tJ2LCTxUTanW4z/FmNB7l327UfMq9A= +sigs.k8s.io/controller-runtime v0.22.4/go.mod h1:+QX1XUpTXN4mLoblf4tqr5CQcyHPAki2HLXqQMY6vh8= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/client-go/hack/boilerplate.go.txt b/client-go/hack/boilerplate.go.txt new file mode 100644 index 000000000..e1732e8d5 --- /dev/null +++ b/client-go/hack/boilerplate.go.txt @@ -0,0 +1,13 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. diff --git a/client-go/hack/update-codegen.sh b/client-go/hack/update-codegen.sh new file mode 100755 index 000000000..8e6f8e6a9 --- /dev/null +++ b/client-go/hack/update-codegen.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)" +CODEGEN_ROOT="${REPO_ROOT}/code-generator" + +export KUBE_CODEGEN_ROOT="${CODEGEN_ROOT}" + +source "${CODEGEN_ROOT}/kube_codegen.sh" + +kube::codegen::gen_client \ + --proto-base "github.com/nvidia/nvsentinel/api/gen/go" \ + --output-dir "${REPO_ROOT}/client-go" \ + --output-pkg "github.com/nvidia/nvsentinel/client-go" \ + --boilerplate "${REPO_ROOT}/client-go/hack/boilerplate.go.txt" \ + --clientset-name "client" \ + --versioned-name "versioned" \ + --with-watch \ + --listers-name "listers" \ + --informers-name "informers" \ + "${REPO_ROOT}/api" diff --git a/client-go/informers/externalversions/device/interface.go b/client-go/informers/externalversions/device/interface.go new file mode 100644 index 000000000..661ca6287 --- /dev/null +++ b/client-go/informers/externalversions/device/interface.go @@ -0,0 +1,44 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by informer-gen. DO NOT EDIT. + +package device + +import ( + v1alpha1 "github.com/nvidia/nvsentinel/client-go/informers/externalversions/device/v1alpha1" + internalinterfaces "github.com/nvidia/nvsentinel/client-go/informers/externalversions/internalinterfaces" +) + +// Interface provides access to each of this group's versions. +type Interface interface { + // V1alpha1 provides access to shared informers for resources in V1alpha1. + V1alpha1() v1alpha1.Interface +} + +type group struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// V1alpha1 returns a new v1alpha1.Interface. +func (g *group) V1alpha1() v1alpha1.Interface { + return v1alpha1.New(g.factory, g.namespace, g.tweakListOptions) +} diff --git a/client-go/informers/externalversions/device/v1alpha1/gpu.go b/client-go/informers/externalversions/device/v1alpha1/gpu.go new file mode 100644 index 000000000..901280510 --- /dev/null +++ b/client-go/informers/externalversions/device/v1alpha1/gpu.go @@ -0,0 +1,99 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + context "context" + time "time" + + apidevicev1alpha1 "github.com/nvidia/nvsentinel/api/device/v1alpha1" + versioned "github.com/nvidia/nvsentinel/client-go/client/versioned" + internalinterfaces "github.com/nvidia/nvsentinel/client-go/informers/externalversions/internalinterfaces" + devicev1alpha1 "github.com/nvidia/nvsentinel/client-go/listers/device/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// GPUInformer provides access to a shared informer and lister for +// GPUs. +type GPUInformer interface { + Informer() cache.SharedIndexInformer + Lister() devicev1alpha1.GPULister +} + +type gPUInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// NewGPUInformer constructs a new informer for GPU type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewGPUInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredGPUInformer(client, resyncPeriod, indexers, nil) +} + +// NewFilteredGPUInformer constructs a new informer for GPU type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredGPUInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.DeviceV1alpha1().GPUs().List(context.Background(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.DeviceV1alpha1().GPUs().Watch(context.Background(), options) + }, + ListWithContextFunc: func(ctx context.Context, options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.DeviceV1alpha1().GPUs().List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.DeviceV1alpha1().GPUs().Watch(ctx, options) + }, + }, + &apidevicev1alpha1.GPU{}, + resyncPeriod, + indexers, + ) +} + +func (f *gPUInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredGPUInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *gPUInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apidevicev1alpha1.GPU{}, f.defaultInformer) +} + +func (f *gPUInformer) Lister() devicev1alpha1.GPULister { + return devicev1alpha1.NewGPULister(f.Informer().GetIndexer()) +} diff --git a/client-go/informers/externalversions/device/v1alpha1/interface.go b/client-go/informers/externalversions/device/v1alpha1/interface.go new file mode 100644 index 000000000..1681622f9 --- /dev/null +++ b/client-go/informers/externalversions/device/v1alpha1/interface.go @@ -0,0 +1,43 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + internalinterfaces "github.com/nvidia/nvsentinel/client-go/informers/externalversions/internalinterfaces" +) + +// Interface provides access to all the informers in this group version. +type Interface interface { + // GPUs returns a GPUInformer. + GPUs() GPUInformer +} + +type version struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// GPUs returns a GPUInformer. +func (v *version) GPUs() GPUInformer { + return &gPUInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} +} diff --git a/client-go/informers/externalversions/factory.go b/client-go/informers/externalversions/factory.go new file mode 100644 index 000000000..b2c78b7ba --- /dev/null +++ b/client-go/informers/externalversions/factory.go @@ -0,0 +1,260 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by informer-gen. DO NOT EDIT. + +package externalversions + +import ( + reflect "reflect" + sync "sync" + time "time" + + versioned "github.com/nvidia/nvsentinel/client-go/client/versioned" + device "github.com/nvidia/nvsentinel/client-go/informers/externalversions/device" + internalinterfaces "github.com/nvidia/nvsentinel/client-go/informers/externalversions/internalinterfaces" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + cache "k8s.io/client-go/tools/cache" +) + +// SharedInformerOption defines the functional option type for SharedInformerFactory. +type SharedInformerOption func(*sharedInformerFactory) *sharedInformerFactory + +type sharedInformerFactory struct { + client versioned.Interface + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc + lock sync.Mutex + defaultResync time.Duration + customResync map[reflect.Type]time.Duration + transform cache.TransformFunc + + informers map[reflect.Type]cache.SharedIndexInformer + // startedInformers is used for tracking which informers have been started. + // This allows Start() to be called multiple times safely. + startedInformers map[reflect.Type]bool + // wg tracks how many goroutines were started. + wg sync.WaitGroup + // shuttingDown is true when Shutdown has been called. It may still be running + // because it needs to wait for goroutines. + shuttingDown bool +} + +// WithCustomResyncConfig sets a custom resync period for the specified informer types. +func WithCustomResyncConfig(resyncConfig map[v1.Object]time.Duration) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + for k, v := range resyncConfig { + factory.customResync[reflect.TypeOf(k)] = v + } + return factory + } +} + +// WithTweakListOptions sets a custom filter on all listers of the configured SharedInformerFactory. +func WithTweakListOptions(tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.tweakListOptions = tweakListOptions + return factory + } +} + +// WithNamespace limits the SharedInformerFactory to the specified namespace. +func WithNamespace(namespace string) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.namespace = namespace + return factory + } +} + +// WithTransform sets a transform on all informers. +func WithTransform(transform cache.TransformFunc) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.transform = transform + return factory + } +} + +// NewSharedInformerFactory constructs a new instance of sharedInformerFactory for all namespaces. +func NewSharedInformerFactory(client versioned.Interface, defaultResync time.Duration) SharedInformerFactory { + return NewSharedInformerFactoryWithOptions(client, defaultResync) +} + +// NewFilteredSharedInformerFactory constructs a new instance of sharedInformerFactory. +// Listers obtained via this SharedInformerFactory will be subject to the same filters +// as specified here. +// Deprecated: Please use NewSharedInformerFactoryWithOptions instead +func NewFilteredSharedInformerFactory(client versioned.Interface, defaultResync time.Duration, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerFactory { + return NewSharedInformerFactoryWithOptions(client, defaultResync, WithNamespace(namespace), WithTweakListOptions(tweakListOptions)) +} + +// NewSharedInformerFactoryWithOptions constructs a new instance of a SharedInformerFactory with additional options. +func NewSharedInformerFactoryWithOptions(client versioned.Interface, defaultResync time.Duration, options ...SharedInformerOption) SharedInformerFactory { + factory := &sharedInformerFactory{ + client: client, + namespace: v1.NamespaceAll, + defaultResync: defaultResync, + informers: make(map[reflect.Type]cache.SharedIndexInformer), + startedInformers: make(map[reflect.Type]bool), + customResync: make(map[reflect.Type]time.Duration), + } + + // Apply all options + for _, opt := range options { + factory = opt(factory) + } + + return factory +} + +func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) { + f.lock.Lock() + defer f.lock.Unlock() + + if f.shuttingDown { + return + } + + for informerType, informer := range f.informers { + if !f.startedInformers[informerType] { + f.wg.Add(1) + // We need a new variable in each loop iteration, + // otherwise the goroutine would use the loop variable + // and that keeps changing. + informer := informer + go func() { + defer f.wg.Done() + informer.Run(stopCh) + }() + f.startedInformers[informerType] = true + } + } +} + +func (f *sharedInformerFactory) Shutdown() { + f.lock.Lock() + f.shuttingDown = true + f.lock.Unlock() + + // Will return immediately if there is nothing to wait for. + f.wg.Wait() +} + +func (f *sharedInformerFactory) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool { + informers := func() map[reflect.Type]cache.SharedIndexInformer { + f.lock.Lock() + defer f.lock.Unlock() + + informers := map[reflect.Type]cache.SharedIndexInformer{} + for informerType, informer := range f.informers { + if f.startedInformers[informerType] { + informers[informerType] = informer + } + } + return informers + }() + + res := map[reflect.Type]bool{} + for informType, informer := range informers { + res[informType] = cache.WaitForCacheSync(stopCh, informer.HasSynced) + } + return res +} + +// InformerFor returns the SharedIndexInformer for obj using an internal +// client. +func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer { + f.lock.Lock() + defer f.lock.Unlock() + + informerType := reflect.TypeOf(obj) + informer, exists := f.informers[informerType] + if exists { + return informer + } + + resyncPeriod, exists := f.customResync[informerType] + if !exists { + resyncPeriod = f.defaultResync + } + + informer = newFunc(f.client, resyncPeriod) + informer.SetTransform(f.transform) + f.informers[informerType] = informer + + return informer +} + +// SharedInformerFactory provides shared informers for resources in all known +// API group versions. +// +// It is typically used like this: +// +// ctx, cancel := context.Background() +// defer cancel() +// factory := NewSharedInformerFactory(client, resyncPeriod) +// defer factory.WaitForStop() // Returns immediately if nothing was started. +// genericInformer := factory.ForResource(resource) +// typedInformer := factory.SomeAPIGroup().V1().SomeType() +// factory.Start(ctx.Done()) // Start processing these informers. +// synced := factory.WaitForCacheSync(ctx.Done()) +// for v, ok := range synced { +// if !ok { +// fmt.Fprintf(os.Stderr, "caches failed to sync: %v", v) +// return +// } +// } +// +// // Creating informers can also be created after Start, but then +// // Start must be called again: +// anotherGenericInformer := factory.ForResource(resource) +// factory.Start(ctx.Done()) +type SharedInformerFactory interface { + internalinterfaces.SharedInformerFactory + + // Start initializes all requested informers. They are handled in goroutines + // which run until the stop channel gets closed. + // Warning: Start does not block. When run in a go-routine, it will race with a later WaitForCacheSync. + Start(stopCh <-chan struct{}) + + // Shutdown marks a factory as shutting down. At that point no new + // informers can be started anymore and Start will return without + // doing anything. + // + // In addition, Shutdown blocks until all goroutines have terminated. For that + // to happen, the close channel(s) that they were started with must be closed, + // either before Shutdown gets called or while it is waiting. + // + // Shutdown may be called multiple times, even concurrently. All such calls will + // block until all goroutines have terminated. + Shutdown() + + // WaitForCacheSync blocks until all started informers' caches were synced + // or the stop channel gets closed. + WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool + + // ForResource gives generic access to a shared informer of the matching type. + ForResource(resource schema.GroupVersionResource) (GenericInformer, error) + + // InformerFor returns the SharedIndexInformer for obj using an internal + // client. + InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer + + Device() device.Interface +} + +func (f *sharedInformerFactory) Device() device.Interface { + return device.New(f, f.namespace, f.tweakListOptions) +} diff --git a/client-go/informers/externalversions/generic.go b/client-go/informers/externalversions/generic.go new file mode 100644 index 000000000..f8ccccacc --- /dev/null +++ b/client-go/informers/externalversions/generic.go @@ -0,0 +1,60 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by informer-gen. DO NOT EDIT. + +package externalversions + +import ( + fmt "fmt" + + v1alpha1 "github.com/nvidia/nvsentinel/api/device/v1alpha1" + schema "k8s.io/apimachinery/pkg/runtime/schema" + cache "k8s.io/client-go/tools/cache" +) + +// GenericInformer is type of SharedIndexInformer which will locate and delegate to other +// sharedInformers based on type +type GenericInformer interface { + Informer() cache.SharedIndexInformer + Lister() cache.GenericLister +} + +type genericInformer struct { + informer cache.SharedIndexInformer + resource schema.GroupResource +} + +// Informer returns the SharedIndexInformer. +func (f *genericInformer) Informer() cache.SharedIndexInformer { + return f.informer +} + +// Lister returns the GenericLister. +func (f *genericInformer) Lister() cache.GenericLister { + return cache.NewGenericLister(f.Informer().GetIndexer(), f.resource) +} + +// ForResource gives generic access to a shared informer of the matching type +// TODO extend this to unknown resources with a client pool +func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { + switch resource { + // Group=device.nvidia.com, Version=v1alpha1 + case v1alpha1.SchemeGroupVersion.WithResource("gpus"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Device().V1alpha1().GPUs().Informer()}, nil + + } + + return nil, fmt.Errorf("no informer found for %v", resource) +} diff --git a/client-go/informers/externalversions/internalinterfaces/factory_interfaces.go b/client-go/informers/externalversions/internalinterfaces/factory_interfaces.go new file mode 100644 index 000000000..ba0352657 --- /dev/null +++ b/client-go/informers/externalversions/internalinterfaces/factory_interfaces.go @@ -0,0 +1,38 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by informer-gen. DO NOT EDIT. + +package internalinterfaces + +import ( + time "time" + + versioned "github.com/nvidia/nvsentinel/client-go/client/versioned" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + cache "k8s.io/client-go/tools/cache" +) + +// NewInformerFunc takes versioned.Interface and time.Duration to return a SharedIndexInformer. +type NewInformerFunc func(versioned.Interface, time.Duration) cache.SharedIndexInformer + +// SharedInformerFactory a small interface to allow for adding an informer without an import cycle +type SharedInformerFactory interface { + Start(stopCh <-chan struct{}) + InformerFor(obj runtime.Object, newFunc NewInformerFunc) cache.SharedIndexInformer +} + +// TweakListOptionsFunc is a function that transforms a v1.ListOptions. +type TweakListOptionsFunc func(*v1.ListOptions) diff --git a/client-go/listers/device/v1alpha1/expansion_generated.go b/client-go/listers/device/v1alpha1/expansion_generated.go new file mode 100644 index 000000000..1aa65cee4 --- /dev/null +++ b/client-go/listers/device/v1alpha1/expansion_generated.go @@ -0,0 +1,21 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +// GPUListerExpansion allows custom methods to be added to +// GPULister. +type GPUListerExpansion interface{} diff --git a/client-go/listers/device/v1alpha1/gpu.go b/client-go/listers/device/v1alpha1/gpu.go new file mode 100644 index 000000000..709bd429f --- /dev/null +++ b/client-go/listers/device/v1alpha1/gpu.go @@ -0,0 +1,46 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + devicev1alpha1 "github.com/nvidia/nvsentinel/api/device/v1alpha1" + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" +) + +// GPULister helps list GPUs. +// All objects returned here must be treated as read-only. +type GPULister interface { + // List lists all GPUs in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*devicev1alpha1.GPU, err error) + // Get retrieves the GPU from the index for a given name. + // Objects returned here must be treated as read-only. + Get(name string) (*devicev1alpha1.GPU, error) + GPUListerExpansion +} + +// gPULister implements the GPULister interface. +type gPULister struct { + listers.ResourceIndexer[*devicev1alpha1.GPU] +} + +// NewGPULister returns a new GPULister. +func NewGPULister(indexer cache.Indexer) GPULister { + return &gPULister{listers.New[*devicev1alpha1.GPU](indexer, devicev1alpha1.Resource("gpu"))} +} diff --git a/client-go/nvgrpc/client_conn.go b/client-go/nvgrpc/client_conn.go new file mode 100644 index 000000000..0aedc8884 --- /dev/null +++ b/client-go/nvgrpc/client_conn.go @@ -0,0 +1,78 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvgrpc + +import ( + "fmt" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/keepalive" +) + +// ClientConnFor creates a new gRPC connection using the provided configuration and options. +func ClientConnFor(config *Config, opts ...DialOption) (*grpc.ClientConn, error) { + if config == nil { + return nil, fmt.Errorf("config cannot be nil") + } + + cfg := *config // Shallow copy to avoid mutation + + dOpts := &dialOptions{} + for _, opt := range opts { + opt(dOpts) + } + + cfg.logger = dOpts.logger + + cfg.Default() + + if err := cfg.Validate(); err != nil { + return nil, err + } + + logger := cfg.GetLogger() + + grpcOpts := []grpc.DialOption{ + grpc.WithUserAgent(cfg.UserAgent), + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: DefaultKeepAliveTime, + Timeout: DefaultKeepAliveTimeout, + PermitWithoutStream: true, // Allow keepalive pings even with no active RPCs. + }), + } + + // Build the unary interceptor chain. + unaryInterceptors := []grpc.UnaryClientInterceptor{ + NewLatencyUnaryInterceptor(logger), + } + unaryInterceptors = append(unaryInterceptors, dOpts.unaryInterceptors...) + grpcOpts = append(grpcOpts, grpc.WithChainUnaryInterceptor(unaryInterceptors...)) + + // Build the stream interceptor chain. + streamInterceptors := []grpc.StreamClientInterceptor{ + NewLatencyStreamInterceptor(logger), + } + streamInterceptors = append(streamInterceptors, dOpts.streamInterceptors...) + grpcOpts = append(grpcOpts, grpc.WithChainStreamInterceptor(streamInterceptors...)) + + conn, err := grpc.NewClient(cfg.Target, grpcOpts...) + if err != nil { + return nil, fmt.Errorf("failed to create gRPC client for %s: %w", cfg.Target, err) + } + + return conn, nil +} diff --git a/client-go/nvgrpc/client_conn_test.go b/client-go/nvgrpc/client_conn_test.go new file mode 100644 index 000000000..06b6a1c12 --- /dev/null +++ b/client-go/nvgrpc/client_conn_test.go @@ -0,0 +1,57 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvgrpc + +import ( + "testing" + + "github.com/go-logr/logr" +) + +func TestClientConnFor(t *testing.T) { + t.Run("Config defaulting does not mutate original", func(t *testing.T) { + cfg := &Config{} + conn, err := ClientConnFor(cfg) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer conn.Close() + + if cfg.Target != "" { + t.Errorf("Original config was mutated! Target is %q", cfg.Target) + } + }) + + t.Run("Config.Default() works correctly", func(t *testing.T) { + cfg := &Config{} + cfg.Default() + + if cfg.Target == "" { + t.Error("Target was not defaulted") + } + if cfg.UserAgent == "" { + t.Error("UserAgent was not defaulted") + } + }) + + t.Run("Client creation respects WithLogger option", func(t *testing.T) { + cfg := &Config{Target: "unix:///tmp/test.sock"} + conn, err := ClientConnFor(cfg, WithLogger(logr.Discard())) + if err != nil { + t.Fatalf("failed to create client: %v", err) + } + conn.Close() + }) +} diff --git a/client-go/nvgrpc/config.go b/client-go/nvgrpc/config.go new file mode 100644 index 000000000..cc23ac1b4 --- /dev/null +++ b/client-go/nvgrpc/config.go @@ -0,0 +1,90 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvgrpc + +import ( + "fmt" + "os" + "time" + + "github.com/go-logr/logr" + "github.com/nvidia/nvsentinel/client-go/version" +) + +const ( + // NvidiaDeviceAPITargetEnvVar is the environment variable that overrides the gRPC target. + NvidiaDeviceAPITargetEnvVar = "NVIDIA_DEVICE_API_TARGET" + + // DefaultNvidiaDeviceAPISocket is the default Unix domain socket path. + DefaultNvidiaDeviceAPISocket = "unix:///var/run/nvidia-device-api/device-api.sock" + + // DefaultKeepAliveTime is the default frequency of keepalive pings. + DefaultKeepAliveTime = 5 * time.Minute + + // DefaultKeepAliveTimeout is the default time to wait for a keepalive pong. + DefaultKeepAliveTimeout = 20 * time.Second +) + +// Config holds configuration for the Device API client. +type Config struct { + // Target is the address of the gRPC server (e.g. "unix:///path/to/socket"). + Target string + + // UserAgent is the string to use for the gRPC User-Agent header. + UserAgent string + + logger logr.Logger +} + +// Default populates unset fields in the Config with default values. +func (c *Config) Default() { + if c.Target == "" { + c.Target = os.Getenv(NvidiaDeviceAPITargetEnvVar) + } + + if c.Target == "" { + c.Target = DefaultNvidiaDeviceAPISocket + } + + if c.UserAgent == "" { + c.UserAgent = version.UserAgent() + } + + if c.logger.GetSink() == nil { + c.logger = logr.Discard() + } +} + +// Validate checks if the Config is valid and returns an error if not. +func (c *Config) Validate() error { + if c.Target == "" { + return fmt.Errorf("gRPC target address is required; verify %s is not empty", NvidiaDeviceAPITargetEnvVar) + } + + if c.UserAgent == "" { + return fmt.Errorf("user-agent cannot be empty") + } + + return nil +} + +// GetLogger returns the configured logger. If no logger is set, it returns a discard logger. +func (c *Config) GetLogger() logr.Logger { + if c.logger.GetSink() == nil { + return logr.Discard() + } + + return c.logger +} diff --git a/client-go/nvgrpc/config_test.go b/client-go/nvgrpc/config_test.go new file mode 100644 index 000000000..437afafb0 --- /dev/null +++ b/client-go/nvgrpc/config_test.go @@ -0,0 +1,120 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package nvgrpc + +import ( + "testing" +) + +func TestConfig_Default_TargetPrecedence(t *testing.T) { + tests := []struct { + name string + argTarget string + envTarget string + wantTarget string + }{ + { + name: "Explicit target is preserved", + argTarget: "unix:///arg.sock", + envTarget: "unix:///env.sock", + wantTarget: "unix:///arg.sock", + }, + { + name: "Env var used when target is empty", + argTarget: "", + envTarget: "unix:///env.sock", + wantTarget: "unix:///env.sock", + }, + { + name: "Default used when both are empty", + argTarget: "", + envTarget: "", + wantTarget: DefaultNvidiaDeviceAPISocket, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv(NvidiaDeviceAPITargetEnvVar, tt.envTarget) + + cfg := &Config{Target: tt.argTarget} + cfg.Default() + + if cfg.Target != tt.wantTarget { + t.Errorf("Target = %q, want %q", cfg.Target, tt.wantTarget) + } + }) + } +} + +func TestConfig_Default_UserAgent(t *testing.T) { + t.Run("Populates default UserAgent if empty", func(t *testing.T) { + cfg := &Config{} + cfg.Default() + + if cfg.UserAgent == "" { + t.Error("UserAgent should have been populated with version-based default") + } + }) + + t.Run("Preserves custom UserAgent", func(t *testing.T) { + custom := "my-custom-agent/1.0" + cfg := &Config{UserAgent: custom} + cfg.Default() + + if cfg.UserAgent != custom { + t.Errorf("UserAgent = %q, want %q", cfg.UserAgent, custom) + } + }) +} + +func TestConfig_Validate(t *testing.T) { + tests := []struct { + name string + cfg Config + wantErr bool + }{ + { + name: "Valid config", + cfg: Config{ + Target: "unix:///var/run/test.sock", + UserAgent: "test/1.0", + }, + wantErr: false, + }, + { + name: "Missing target", + cfg: Config{ + UserAgent: "test/1.0", + }, + wantErr: true, + }, + { + name: "Missing user agent", + cfg: Config{ + Target: "unix:///var/run/test.sock", + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.cfg.Validate() + if (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/client-go/nvgrpc/interceptors.go b/client-go/nvgrpc/interceptors.go new file mode 100644 index 000000000..433f5c499 --- /dev/null +++ b/client-go/nvgrpc/interceptors.go @@ -0,0 +1,95 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvgrpc + +import ( + "context" + "time" + + "github.com/go-logr/logr" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// NewLatencyUnaryInterceptor returns an interceptor that logs the latency and status of unary RPCs. +func NewLatencyUnaryInterceptor(logger logr.Logger) grpc.UnaryClientInterceptor { + return func(ctx context.Context, method string, req, reply interface{}, + cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + start := time.Now() + err := invoker(ctx, method, req, reply, cc, opts...) + duration := time.Since(start) + + s := status.Convert(err) + code := s.Code() + kv := []interface{}{ + "grpc.method", method, + "duration", duration, + "code", int(code), + } + + if err != nil { + if code == codes.Canceled || code == codes.DeadlineExceeded { + logger.V(4).Info("RPC finished with context error", kv...) + return err + } + + logger.Error(err, "RPC failed", kv...) + + return err + } + + if logger.V(6).Enabled() { + logger.V(6).Info("RPC succeeded", kv...) + } + + return nil + } +} + +// NewLatencyStreamInterceptor returns an interceptor that logs the latency and status of stream establishment. +func NewLatencyStreamInterceptor(logger logr.Logger) grpc.StreamClientInterceptor { + return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, + method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + start := time.Now() + stream, err := streamer(ctx, desc, cc, method, opts...) + duration := time.Since(start) + + s := status.Convert(err) + code := s.Code() + kv := []interface{}{ + "grpc.method", method, + "duration", duration, + "code", int(code), + } + + if err != nil { + if code == codes.Canceled || code == codes.DeadlineExceeded { + logger.V(4).Info("Stream establishment canceled", kv...) + return stream, err + } + + logger.Error(err, "Stream establishment failed", kv...) + + return stream, err + } + + if logger.V(4).Enabled() { + logger.V(4).Info("Stream started", kv...) + } + + return stream, nil + } +} diff --git a/client-go/nvgrpc/interceptors_test.go b/client-go/nvgrpc/interceptors_test.go new file mode 100644 index 000000000..47b9b9b16 --- /dev/null +++ b/client-go/nvgrpc/interceptors_test.go @@ -0,0 +1,103 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvgrpc + +import ( + "context" + "errors" + "testing" + "time" + + logr "github.com/go-logr/logr/testing" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestLatencyUnaryInterceptor(t *testing.T) { + tests := []struct { + name string + method string + log bool + invokerErr error + expectedCode codes.Code + }{ + {"Returns OK code for successful call", "/svc/success", true, nil, codes.OK}, + {"Returns internal status error on internal error", "/svc/internal_error", true, status.Error(codes.Internal, "fail"), codes.Internal}, + {"Returns canceled status error when canceled", "/svc/cancel", true, status.Error(codes.Canceled, ""), codes.Canceled}, + {"Returns deadline exceeded status error on timeout", "/svc/timeout", true, status.Error(codes.DeadlineExceeded, ""), codes.DeadlineExceeded}, + {"Does not log if level too low", "/svc/skip_log", false, nil, codes.OK}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + logger := logr.NewTestLogger(t) + if tt.log { + logger = logger.V(4) + } + + interceptor := NewLatencyUnaryInterceptor(logger) + + invoker := func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, opts ...grpc.CallOption) error { + if !tt.log { + time.Sleep(1 * time.Millisecond) + } + return tt.invokerErr + } + + err := interceptor(context.Background(), tt.method, nil, nil, nil, invoker) + if !errors.Is(err, tt.invokerErr) { + t.Fatalf("Returned error mismatch. Got %v, want %v", err, tt.invokerErr) + } + }) + } +} + +func TestLatencyStreamInterceptor(t *testing.T) { + tests := []struct { + name string + method string + log bool + streamerErr error + }{ + {"Returns nil on successful start of stream", "/svc/start_stream", true, nil}, + {"Returns internal status error for failed stream", "/svc/stream_fail", true, status.Error(codes.Internal, "fail")}, + {"Does not log if level too low", "/svc/skip_log", false, nil}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + logger := logr.NewTestLogger(t) + if tt.log { + logger = logger.V(4) + } + + interceptor := NewLatencyStreamInterceptor(logger) + desc := &grpc.StreamDesc{} + + streamer := func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, opts ...grpc.CallOption) (grpc.ClientStream, error) { + if !tt.log { + time.Sleep(1 * time.Millisecond) + } + return nil, tt.streamerErr + } + + _, err := interceptor(context.Background(), desc, nil, tt.method, streamer) + if !errors.Is(err, tt.streamerErr) { + t.Fatalf("Returned error mismatch. Got %v, want %v", err, tt.streamerErr) + } + }) + } +} diff --git a/client-go/nvgrpc/options.go b/client-go/nvgrpc/options.go new file mode 100644 index 000000000..d25991a50 --- /dev/null +++ b/client-go/nvgrpc/options.go @@ -0,0 +1,50 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvgrpc + +import ( + "github.com/go-logr/logr" + "google.golang.org/grpc" +) + +// DialOption configures the gRPC client connection. +type DialOption func(*dialOptions) + +type dialOptions struct { + logger logr.Logger + unaryInterceptors []grpc.UnaryClientInterceptor + streamInterceptors []grpc.StreamClientInterceptor +} + +// WithLogger sets the logger to be used by the client. +func WithLogger(logger logr.Logger) DialOption { + return func(opts *dialOptions) { + opts.logger = logger + } +} + +// WithUnaryInterceptor adds a unary client interceptor to the chain. +func WithUnaryInterceptor(interceptor grpc.UnaryClientInterceptor) DialOption { + return func(opts *dialOptions) { + opts.unaryInterceptors = append(opts.unaryInterceptors, interceptor) + } +} + +// WithStreamInterceptor adds a stream client interceptor to the chain. +func WithStreamInterceptor(interceptor grpc.StreamClientInterceptor) DialOption { + return func(opts *dialOptions) { + opts.streamInterceptors = append(opts.streamInterceptors, interceptor) + } +} diff --git a/client-go/nvgrpc/options_test.go b/client-go/nvgrpc/options_test.go new file mode 100644 index 000000000..43559dc00 --- /dev/null +++ b/client-go/nvgrpc/options_test.go @@ -0,0 +1,65 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvgrpc + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + "google.golang.org/grpc" +) + +func TestDialOptions(t *testing.T) { + testLogger := logr.Discard() + + dummyUnary := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + return invoker(ctx, method, req, reply, cc, opts...) + } + dummyStream := func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + return streamer(ctx, desc, cc, method, opts...) + } + + opts := []DialOption{ + WithLogger(testLogger), + WithUnaryInterceptor(dummyUnary), + WithStreamInterceptor(dummyStream), + WithUnaryInterceptor(dummyUnary), // Test multiple appends + WithStreamInterceptor(dummyStream), + } + + dOpts := &dialOptions{} + for _, opt := range opts { + opt(dOpts) + } + + t.Run("Logger is correctly assigned", func(t *testing.T) { + if dOpts.logger != testLogger { + t.Errorf("expected logger to be set, got %v", dOpts.logger) + } + }) + + t.Run("Unary interceptors are correctly appended", func(t *testing.T) { + if len(dOpts.unaryInterceptors) != 2 { + t.Errorf("expected 2 unary interceptors, got %d", len(dOpts.unaryInterceptors)) + } + }) + + t.Run("Stream interceptors are correctly appended", func(t *testing.T) { + if len(dOpts.streamInterceptors) != 2 { + t.Errorf("expected 2 stream interceptors, got %d", len(dOpts.streamInterceptors)) + } + }) +} diff --git a/client-go/nvgrpc/watcher.go b/client-go/nvgrpc/watcher.go new file mode 100644 index 000000000..226493083 --- /dev/null +++ b/client-go/nvgrpc/watcher.go @@ -0,0 +1,185 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvgrpc + +import ( + "context" + "errors" + "io" + "sync" + + "github.com/go-logr/logr" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" +) + +// Source defines the interface for reading events from a source. +type Source interface { + Next() (eventType string, obj runtime.Object, err error) + Close() error +} + +// Watcher implements watch.Interface. +type Watcher struct { + cancel context.CancelFunc // cancels the event source + result chan watch.Event // channel delivering watch events + source Source // the underlying event source + done chan struct{} // closed when watcher stops + stopOnce sync.Once // ensures stop is idempotent + logger logr.Logger +} + +// NewWatcher creates a Watcher and starts receiving events. +func NewWatcher( + source Source, + cancel context.CancelFunc, + logger logr.Logger, +) watch.Interface { + w := &Watcher{ + cancel: cancel, + result: make(chan watch.Event, 100), + source: source, + done: make(chan struct{}), + logger: logger.WithName("watcher"), + } + + go w.receive() + + return w +} + +// Stop cancels the context and closes the event source. +func (w *Watcher) Stop() { + w.stopOnce.Do(func() { + w.logger.V(4).Info("Stopping watcher") + w.cancel() + + if err := w.source.Close(); err != nil { + w.logger.V(4).Info("Error closing source during stop", "err", err) + } + + close(w.done) + }) +} + +// ResultChan returns the channel delivering watch events. +func (w *Watcher) ResultChan() <-chan watch.Event { + return w.result +} + +// receive reads events from the source and sends them to result channel. +// +// nolint:cyclop // Complexity is necessary to handle various gRPC stream states and event types. +func (w *Watcher) receive() { + defer func() { + w.logger.V(4).Info("Watcher receive loop exiting") + close(w.result) + }() + defer w.Stop() + + for { + w.logger.V(6).Info("Waiting for next event from source") + + typeStr, obj, err := w.source.Next() + if err != nil { + if errors.Is(err, io.EOF) || status.Code(err) == codes.Canceled { + w.logger.V(3).Info("Watch stream closed normally") + return + } + + w.logger.Error(err, "Watch stream encountered unexpected error") + w.sendError(err) + + return + } + + var eventType watch.EventType + + switch typeStr { + case "ADDED": + eventType = watch.Added + case "MODIFIED": + eventType = watch.Modified + case "DELETED": + eventType = watch.Deleted + case "ERROR": + w.logger.V(4).Info("Received explicit ERROR event from server") + + w.result <- watch.Event{Type: watch.Error, Object: obj} + + return + default: + w.logger.V(2).Info("Skipping unknown event type from server", "rawType", typeStr) + continue + } + + select { + case <-w.done: + w.logger.V(3).Info("Watcher stopping; aborting receive loop") + return + case w.result <- watch.Event{Type: eventType, Object: obj}: + if meta, ok := obj.(metav1.Object); ok { + w.logger.V(6).Info("Event dispatched to Informer", + "type", eventType, + "name", meta.GetName(), + "resourceVersion", meta.GetResourceVersion(), + ) + } + } + } +} + +func (w *Watcher) sendError(err error) { + st := status.Convert(err) + + code := st.Code() + statusErr := &metav1.Status{ + Status: metav1.StatusFailure, + Message: st.Message(), + Code: int32(code), // #nosec G115 + } + + //nolint:exhaustive // Only specific gRPC codes require special Kubernetes status mapping. + switch code { + case codes.OutOfRange, codes.ResourceExhausted, codes.InvalidArgument: + // CRITICAL for Informers: This tells the Reflector to perform a new List operation. + statusErr.Reason = metav1.StatusReasonExpired + statusErr.Code = 410 + case codes.PermissionDenied: + statusErr.Reason = metav1.StatusReasonForbidden + statusErr.Code = 403 + case codes.NotFound: + statusErr.Reason = metav1.StatusReasonNotFound + statusErr.Code = 404 + default: + w.logger.V(5).Info("Using default status mapping for gRPC code", "code", code) + } + + w.logger.V(4).Info("Sending error event to informer", + "code", statusErr.Code, + "reason", statusErr.Reason, + "message", statusErr.Message, + ) + + select { + case <-w.done: + w.logger.V(4).Info("Watcher already done, dropping error event") + case w.result <- watch.Event{Type: watch.Error, Object: statusErr}: + } +} diff --git a/client-go/nvgrpc/watcher_test.go b/client-go/nvgrpc/watcher_test.go new file mode 100644 index 000000000..303c49e44 --- /dev/null +++ b/client-go/nvgrpc/watcher_test.go @@ -0,0 +1,239 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvgrpc + +import ( + "context" + "io" + "testing" + "time" + + "github.com/go-logr/logr" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" +) + +func TestWatcher_NormalEvents(t *testing.T) { + _, cancel := context.WithCancel(context.Background()) + defer cancel() + + events := make(chan testEvent, 3) + events <- testEvent{"ADDED", &FakeObject{Name: "obj1"}} + events <- testEvent{"MODIFIED", &FakeObject{Name: "obj1"}} + events <- testEvent{"DELETED", &FakeObject{Name: "obj1"}} + close(events) + + source := &FakeSource{events: events, done: make(chan struct{})} + w := NewWatcher(source, cancel, logr.Discard()) + + var got []watch.Event + for e := range w.ResultChan() { + got = append(got, e) + } + + wantTypes := []watch.EventType{watch.Added, watch.Modified, watch.Deleted} + if len(got) != len(wantTypes) { + t.Fatalf("got %d events, want %d", len(got), len(wantTypes)) + } + for i, ev := range got { + if ev.Type != wantTypes[i] { + t.Errorf("event %d: got type %v, want %v", i, ev.Type, wantTypes[i]) + } + } +} + +func TestWatcher_UnknownEventType_Ignored(t *testing.T) { + _, cancel := context.WithCancel(context.Background()) + defer cancel() + + events := make(chan testEvent, 2) + events <- testEvent{"UNKNOWN", &FakeObject{Name: "obj1"}} + events <- testEvent{"ADDED", &FakeObject{Name: "obj2"}} + close(events) + + source := &FakeSource{events: events, done: make(chan struct{})} + w := NewWatcher(source, cancel, logr.Discard()) + + var got []watch.Event + for e := range w.ResultChan() { + got = append(got, e) + } + + if len(got) != 1 { + t.Fatalf("expected 1 event, got %d", len(got)) + } + if got[0].Type != watch.Added { + t.Errorf("expected ADDED, got %v", got[0].Type) + } +} + +func TestWatcher_Errors(t *testing.T) { + cases := []struct { + name string + err error + wantReason metav1.StatusReason + wantCode int32 + }{ + {"Internal", status.Error(codes.Internal, "err"), "", int32(codes.Internal)}, + {"OutOfRange", status.Error(codes.OutOfRange, "err"), metav1.StatusReasonExpired, 410}, + {"InvalidArgument", status.Error(codes.InvalidArgument, "err"), metav1.StatusReasonExpired, 410}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, cancel := context.WithCancel(context.Background()) + defer cancel() + + source := &FakeSource{errs: []error{tc.err}, done: make(chan struct{})} + w := NewWatcher(source, cancel, logr.Discard()) + + e := <-w.ResultChan() + if e.Type != watch.Error { + t.Fatalf("expected watch.Error, got %v", e.Type) + } + st, ok := e.Object.(*metav1.Status) + if !ok { + t.Fatal("expected metav1.Status object") + } + if st.Reason != tc.wantReason || st.Code != tc.wantCode { + t.Errorf("got %+v, wantReason %v, wantCode %d", st, tc.wantReason, tc.wantCode) + } + }) + } +} + +func TestWatcher_ErrorTerminatesStream(t *testing.T) { + _, cancel := context.WithCancel(context.Background()) + defer cancel() + + source := &FakeSource{ + errs: []error{ + status.Error(codes.Internal, "fatal error"), + status.Error(codes.Internal, "should never be reached"), + }, + done: make(chan struct{}), + } + w := NewWatcher(source, cancel, logr.Discard()) + + count := 0 + timeout := time.After(500 * time.Millisecond) + +Receive: + for { + select { + case _, ok := <-w.ResultChan(): + if !ok { + break Receive + } + count++ + case <-timeout: + t.Fatal("Test timed out waiting for ResultChan to close") + } + } + + if count != 1 { + t.Errorf("expected 1 error event, got %d", count) + } +} + +func TestWatcher_Stop(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + source := NewFakeSource() + + w := NewWatcher(source, cancel, logr.Discard()) + // Allow the receive loop to start + time.Sleep(10 * time.Millisecond) + w.Stop() + + select { + case <-ctx.Done(): + case <-time.After(time.Second): + t.Error("context not cancelled after Stop()") + } + + select { + case _, ok := <-w.ResultChan(): + if ok { + t.Error("ResultChan not closed") + } + case <-time.After(time.Second): + t.Error("ResultChan hang") + } +} + +// FakeObject is a minimal implementation of runtime.Object. +type FakeObject struct { + metav1.TypeMeta + Name string +} + +func (f *FakeObject) DeepCopyObject() runtime.Object { + return &FakeObject{ + TypeMeta: f.TypeMeta, + Name: f.Name, + } +} + +type testEvent struct { + eventType string + obj runtime.Object +} + +// FakeSource implements nvgrpc.Source. +type FakeSource struct { + events chan testEvent + errs []error + done chan struct{} +} + +func NewFakeSource() *FakeSource { + return &FakeSource{ + events: make(chan testEvent, 10), + done: make(chan struct{}), + } +} + +func (f *FakeSource) Next() (string, runtime.Object, error) { + if len(f.errs) > 0 { + err := f.errs[0] + f.errs = f.errs[1:] + return "", nil, err + } + + select { + case <-f.done: + return "", nil, io.EOF + case e, ok := <-f.events: + if !ok { + return "", nil, io.EOF + } + return e.eventType, e.obj, nil + } +} + +func (f *FakeSource) Close() error { + if f.done == nil { + return nil + } + select { + case <-f.done: + default: + close(f.done) + } + return nil +} diff --git a/client-go/tests/device/v1alpha1/clientset_test.go b/client-go/tests/device/v1alpha1/clientset_test.go new file mode 100644 index 000000000..92f6e70e9 --- /dev/null +++ b/client-go/tests/device/v1alpha1/clientset_test.go @@ -0,0 +1,353 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1_test + +import ( + "context" + "fmt" + "net" + "sync" + "testing" + "time" + + devicev1alpha1 "github.com/nvidia/nvsentinel/api/device/v1alpha1" + "github.com/nvidia/nvsentinel/client-go/client/versioned" + "github.com/nvidia/nvsentinel/client-go/client/versioned/scheme" + informers "github.com/nvidia/nvsentinel/client-go/informers/externalversions" + "github.com/nvidia/nvsentinel/client-go/nvgrpc" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/test/bufconn" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + + pb "github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1" +) + +func TestClientset_EndToEnd(t *testing.T) { + lis := bufconn.Listen(1024 * 1024) + s := grpc.NewServer() + + mock := newMockGpuServer() + pb.RegisterGpuServiceServer(s, mock) + + go s.Serve(lis) + defer s.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + dialer := func(context.Context, string) (net.Conn, error) { + return lis.Dial() + } + + conn, err := grpc.DialContext(ctx, "bufconn", + grpc.WithContextDialer(dialer), + grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to dial bufconn: %v", err) + } + defer conn.Close() + + config := &nvgrpc.Config{Target: "passthrough://bufconn"} + cs, err := versioned.NewForConfigAndClient(config, conn) + if err != nil { + t.Fatalf("Failed to create clientset: %v", err) + } + + t.Run("Get", func(t *testing.T) { + gpu, err := cs.DeviceV1alpha1().GPUs().Get(ctx, "gpu-1", metav1.GetOptions{}) + if err != nil { + t.Fatalf("Get failed: %v", err) + } + if gpu.Name != "gpu-1" { + t.Errorf("Expected gpu-1, got %s", gpu.Name) + } + }) + + t.Run("List", func(t *testing.T) { + list, err := cs.DeviceV1alpha1().GPUs().List(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatalf("List failed: %v", err) + } + + if len(list.Items) != 1 { + t.Errorf("Expected 1 item in list, got %d", len(list.Items)) + } + if list.Items[0].ResourceVersion != "7" { + t.Errorf("Expected ResourceVersion 7, got %s", list.Items[0].ResourceVersion) + } + }) + + t.Run("Watch flow with initial snapshot", func(t *testing.T) { + w, err := cs.DeviceV1alpha1().GPUs().Watch(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatalf("Watch failed: %v", err) + } + defer w.Stop() + + // Consume the initial snapshot + select { + case event := <-w.ResultChan(): + if event.Type != watch.Added { + t.Errorf("Expected initial ADDED event, got %v", event.Type) + } + case <-time.After(2 * time.Second): + t.Fatal("Timed out waiting for initial snapshot") + } + + // Trigger event + mock.sendEvent(&pb.WatchGpusResponse{ + Type: "MODIFIED", + Object: &pb.Gpu{ + Metadata: &pb.ObjectMeta{ + Name: "gpu-1", + ResourceVersion: "8", + }, + }, + }) + + select { + case event := <-w.ResultChan(): + if event.Type != watch.Modified { + t.Errorf("Expected event type MODIFIED, got %v", event.Type) + } + + gpu := event.Object.(*devicev1alpha1.GPU) + if gpu.ResourceVersion != "8" { + t.Errorf("Expected ResourceVersion 8, got %s", gpu.ResourceVersion) + } + case <-time.After(2 * time.Second): + t.Fatal("Timed out waiting for modified event") + } + }) + + t.Run("Informer and Lister Sync", func(t *testing.T) { + subCtx, subCancel := context.WithTimeout(ctx, 5*time.Second) + defer subCancel() + + factory := informers.NewSharedInformerFactory(cs, 0) + + // Informer must be instantiated BEFORE starting the factory to register it with the factory. + gpuInformer := factory.Device().V1alpha1().GPUs() + _ = gpuInformer.Informer() + + stopCh := make(chan struct{}) + defer close(stopCh) + + factory.Start(stopCh) + + if !cache.WaitForCacheSync(subCtx.Done(), gpuInformer.Informer().HasSynced) { + t.Fatal("Timed out waiting for cache sync") + } + + // Initial snapshot + lister := gpuInformer.Lister() + gpu, err := lister.Get("gpu-1") + if err != nil { + t.Fatalf("Lister failed to find gpu-1 in cache: %v", err) + } + if gpu.ResourceVersion != "7" { + t.Errorf("Expected cached RV 7, got %s", gpu.ResourceVersion) + } + + // Trigger event + mock.sendEvent(&pb.WatchGpusResponse{ + Type: "MODIFIED", + Object: &pb.Gpu{ + Metadata: &pb.ObjectMeta{ + Name: "gpu-1", + ResourceVersion: "8", + }, + }, + }) + + err = wait.PollUntilContextTimeout(subCtx, 100*time.Millisecond, 3*time.Second, true, func(ctx context.Context) (bool, error) { + updated, err := lister.Get("gpu-1") + if err != nil { + return false, nil + } + return updated.ResourceVersion == "8", nil + }) + + if err != nil { + t.Errorf("Informer failed to update cache with new ResourceVersion: %v", err) + } + }) + + t.Run("Controller-runtime Compatibility", func(t *testing.T) { + factory := informers.NewSharedInformerFactory(cs, 0) + gpuInformer := factory.Device().V1alpha1().GPUs() + + mapper := meta.NewDefaultRESTMapper([]schema.GroupVersion{devicev1alpha1.SchemeGroupVersion}) + mapper.Add(devicev1alpha1.SchemeGroupVersion.WithKind("GPU"), meta.RESTScopeRoot) + + c, err := ctrlcache.New(&rest.Config{Host: "http://localhost:0"}, ctrlcache.Options{ + Scheme: scheme.Scheme, + Mapper: mapper, + NewInformer: func(lw cache.ListerWatcher, obj runtime.Object, resync time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + if _, ok := obj.(*devicev1alpha1.GPU); ok { + return gpuInformer.Informer() + } + return cache.NewSharedIndexInformer(lw, obj, resync, indexers) + }, + }) + if err != nil { + t.Fatalf("Failed to create controller-runtime cache: %v", err) + } + + stopCh := make(chan struct{}) + defer close(stopCh) + + factory.Start(stopCh) + go func() { + if err := c.Start(ctx); err != nil { + if ctx.Err() == nil { + // Errors during Start are expected when context is cancelled during cleanup. + t.Logf("Cache start error (may be expected): %v", err) + } + } + }() + + if !c.WaitForCacheSync(ctx) { + t.Fatal("Controller-runtime cache failed to sync") + } + + // Initial snapshot + var gpu devicev1alpha1.GPU + key := client.ObjectKey{Name: "gpu-1"} + if err := c.Get(ctx, key, &gpu); err != nil { + t.Fatalf("Failed to read initial state from cache: %v", err) + } + if gpu.ResourceVersion != "7" { + t.Errorf("Expected RV 7, got %s", gpu.ResourceVersion) + } + + // Trigger event + mock.sendEvent(&pb.WatchGpusResponse{ + Type: "MODIFIED", + Object: &pb.Gpu{ + Metadata: &pb.ObjectMeta{ + Name: "gpu-1", + ResourceVersion: "8", + }, + }, + }) + + err = wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 2*time.Second, true, func(ctx context.Context) (bool, error) { + var updated devicev1alpha1.GPU + if err := c.Get(ctx, key, &updated); err != nil { + return false, nil + } + return updated.ResourceVersion == "8", nil + }) + + if err != nil { + t.Errorf("Controller-runtime cache failed to reflect gRPC event: %v", err) + } + }) +} + +// --- Mock Server Implementation --- + +type mockGpuServer struct { + pb.UnimplementedGpuServiceServer + mu sync.RWMutex + gpus map[string]*pb.Gpu + watch chan *pb.WatchGpusResponse +} + +func newMockGpuServer() *mockGpuServer { + return &mockGpuServer{ + gpus: map[string]*pb.Gpu{ + "gpu-1": { + Metadata: &pb.ObjectMeta{ + Name: "gpu-1", + ResourceVersion: "7", + }, + Spec: &pb.GpuSpec{Uuid: "GPU-1"}, + }, + }, + watch: make(chan *pb.WatchGpusResponse, 10), + } +} + +func (m *mockGpuServer) GetGpu(ctx context.Context, req *pb.GetGpuRequest) (*pb.GetGpuResponse, error) { + m.mu.RLock() + defer m.mu.RUnlock() + gpu, ok := m.gpus[req.Name] + if !ok { + return nil, fmt.Errorf("%s not found", req.Name) + } + return &pb.GetGpuResponse{Gpu: gpu}, nil +} + +func (m *mockGpuServer) ListGpus(ctx context.Context, req *pb.ListGpusRequest) (*pb.ListGpusResponse, error) { + m.mu.RLock() + defer m.mu.RUnlock() + list := &pb.GpuList{} + for _, g := range m.gpus { + list.Items = append(list.Items, g) + } + return &pb.ListGpusResponse{GpuList: list}, nil +} + +func (m *mockGpuServer) WatchGpus(req *pb.WatchGpusRequest, stream pb.GpuService_WatchGpusServer) error { + m.mu.RLock() + // Send the initial snapshot (Current state) + for _, g := range m.gpus { + select { + case <-stream.Context().Done(): + m.mu.RUnlock() + return nil + default: + if err := stream.Send(&pb.WatchGpusResponse{ + Type: "ADDED", + Object: g, + }); err != nil { + m.mu.RUnlock() + return err + } + } + } + m.mu.RUnlock() + + // Continuous watch (Live events) + for { + select { + case <-stream.Context().Done(): + return nil + case ev, ok := <-m.watch: + if !ok { + return nil + } + if err := stream.Send(ev); err != nil { + return err + } + } + } +} + +func (m *mockGpuServer) sendEvent(ev *pb.WatchGpusResponse) { + m.watch <- ev +} diff --git a/client-go/tests/device/v1alpha1/fake_client_test.go b/client-go/tests/device/v1alpha1/fake_client_test.go new file mode 100644 index 000000000..34755f700 --- /dev/null +++ b/client-go/tests/device/v1alpha1/fake_client_test.go @@ -0,0 +1,103 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1_test + +import ( + "context" + "testing" + "time" + + devicev1alpha1 "github.com/nvidia/nvsentinel/api/device/v1alpha1" + "github.com/nvidia/nvsentinel/client-go/client/versioned/fake" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" +) + +func TestGPUFakeClient(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + gpu1 := &devicev1alpha1.GPU{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gpu-1", + ResourceVersion: "100", + }, + Spec: devicev1alpha1.GPUSpec{UUID: "GPU-1"}, + } + client := fake.NewSimpleClientset(gpu1) + + gpu, err := client.DeviceV1alpha1().GPUs().Get(ctx, "gpu-1", metav1.GetOptions{}) + if err != nil { + t.Errorf("Fake client failed to retrieve GPU: %v", err) + } + if gpu.Name != "gpu-1" { + t.Errorf("Expected %v, got %v", gpu1, gpu) + } + if gpu.ResourceVersion != "100" { + t.Errorf("ResourceVersion mismatch: expected 100, got %s", gpu.ResourceVersion) + } + if gpu.Spec.UUID != "GPU-1" { + t.Errorf("UUID mismatch: expected GPU-1, got %s", gpu.Spec.UUID) + } + + list, err := client.DeviceV1alpha1().GPUs().List(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatalf("Fake client failed to list GPUs: %v", err) + } + if len(list.Items) != 1 { + t.Errorf("Expected 1 GPU, got %d", len(list.Items)) + } + + watchRV := list.ResourceVersion + watcher, err := client.DeviceV1alpha1().GPUs().Watch(ctx, metav1.ListOptions{ + ResourceVersion: watchRV, + }) + if err != nil { + t.Fatalf("Fake client failed to Watch GPUs: %v", err) + } + defer watcher.Stop() + + // Simulate an Event in the background + gpu2 := &devicev1alpha1.GPU{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gpu-2", + ResourceVersion: "101", + }, + Spec: devicev1alpha1.GPUSpec{UUID: "GPU-2"}, + } + go func() { + time.Sleep(100 * time.Millisecond) + client.Tracker().Add(gpu2) + }() + + select { + case event, ok := <-watcher.ResultChan(): + if !ok { + t.Fatal("Watch channel closed prematurely") + } + if event.Type != watch.Added { + t.Errorf("Expected Added event, got %v", event.Type) + } + obj := event.Object.(*devicev1alpha1.GPU) + if obj.Name != "gpu-2" { + t.Errorf("Expected gpu-2, got %s", obj.Name) + } + if obj.ResourceVersion != "101" { + t.Errorf("ResourceVersion mismatch: expected 101, got %s", obj.ResourceVersion) + } + case <-ctx.Done(): + t.Fatal("Timed out waiting for watch event") + } +} diff --git a/client-go/version/version.go b/client-go/version/version.go new file mode 100644 index 000000000..e8d3cb43c --- /dev/null +++ b/client-go/version/version.go @@ -0,0 +1,35 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package version + +import ( + "fmt" + "runtime" +) + +// GitVersion is the semantic version of the client. +// It is set at build time via -ldflags +// e.g., -ldflags "-X 'github.com/nvidia/nvsentinel/client-go/version.GitVersion=v1.2.3'" +var GitVersion = "devel" + +// UserAgent returns the standard user agent string. +func UserAgent() string { + return fmt.Sprintf( + "nvidia-device-api-client/%s (%s/%s)", + GitVersion, + runtime.GOOS, + runtime.GOARCH, + ) +} diff --git a/client-go/version/version_test.go b/client-go/version/version_test.go new file mode 100644 index 000000000..604242567 --- /dev/null +++ b/client-go/version/version_test.go @@ -0,0 +1,33 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package version + +import ( + "fmt" + "runtime" + "testing" +) + +func TestUserAgent(t *testing.T) { + originalVersion := GitVersion + defer func() { GitVersion = originalVersion }() + + GitVersion = "v1.2.3-test" + + expected := fmt.Sprintf("nvidia-device-api-client/v1.2.3-test (%s/%s)", runtime.GOOS, runtime.GOARCH) + if got := UserAgent(); got != expected { + t.Errorf("UserAgent() = %q, want %q", got, expected) + } +} diff --git a/cmd/controller-test/main.go b/cmd/controller-test/main.go new file mode 100644 index 000000000..06e6b10c2 --- /dev/null +++ b/cmd/controller-test/main.go @@ -0,0 +1,178 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Command controller-test runs the NVSentinel controllers locally for integration testing. +// It connects to a remote Kubernetes cluster and processes HealthEvent CRDs. +// +// Usage: +// +// KUBECONFIG=/path/to/kubeconfig controller-test +// controller-test --kubeconfig=/path/to/kubeconfig +package main + +import ( + "context" + "flag" + "os" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + nvsentinelv1alpha1 "github.com/nvidia/nvsentinel/api/nvsentinel/v1alpha1" + "github.com/nvidia/nvsentinel/pkg/controllers/healthevents" +) + +var scheme = runtime.NewScheme() + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(nvsentinelv1alpha1.AddToScheme(scheme)) +} + +func main() { + var ( + metricsAddr string + enableLeaderElection bool + probeAddr string + dryRun bool + ) + + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager.") + flag.BoolVar(&dryRun, "dry-run", false, "Dry run mode - don't actually cordon/drain/remediate") + + klog.InitFlags(nil) + opts := zap.Options{ + Development: true, + } + opts.BindFlags(flag.CommandLine) + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + log := ctrl.Log.WithName("setup") + log.Info("Starting NVSentinel Controller Test Runner", + "dryRun", dryRun, + ) + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + Metrics: metricsserver.Options{BindAddress: metricsAddr}, + HealthProbeBindAddress: probeAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "nvsentinel-controllers.nvidia.com", + }) + if err != nil { + log.Error(err, "Unable to create manager") + os.Exit(1) + } + + // Add field index for spec.nodeName on Pods - required for DrainController + // to list pods by node name using field selectors + ctx := context.Background() + if err := mgr.GetFieldIndexer().IndexField(ctx, &corev1.Pod{}, "spec.nodeName", + func(obj client.Object) []string { + pod := obj.(*corev1.Pod) + if pod.Spec.NodeName == "" { + return nil + } + return []string{pod.Spec.NodeName} + }); err != nil { + log.Error(err, "Unable to create field index for pods") + os.Exit(1) + } + log.Info("Pod field index for spec.nodeName created") + + // Set up QuarantineController + if err := (&healthevents.QuarantineController{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("quarantine-controller"), + MaxConcurrentReconciles: 2, + }).SetupWithManager(mgr); err != nil { + log.Error(err, "Unable to create controller", "controller", "QuarantineController") + os.Exit(1) + } + log.Info("QuarantineController registered") + + // Set up DrainController + if err := (&healthevents.DrainController{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("drain-controller"), + IgnoreDaemonSets: true, + DeleteEmptyDirData: true, + GracePeriodSeconds: 30, + MaxConcurrentReconciles: 2, + }).SetupWithManager(mgr); err != nil { + log.Error(err, "Unable to create controller", "controller", "DrainController") + os.Exit(1) + } + log.Info("DrainController registered") + + // Set up RemediationController + if err := (&healthevents.RemediationController{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("remediation-controller"), + DefaultStrategy: healthevents.StrategyManual, // Default to manual for safety + RebootJobNamespace: "nvsentinel-system", + RebootJobImage: "busybox:latest", + RebootJobTTL: time.Hour, + MaxConcurrentReconciles: 1, + }).SetupWithManager(mgr); err != nil { + log.Error(err, "Unable to create controller", "controller", "RemediationController") + os.Exit(1) + } + log.Info("RemediationController registered") + + // Set up TTLController + if err := (&healthevents.TTLController{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("ttl-controller"), + DefaultTTL: 168 * time.Hour, // 7 days + MaxConcurrentReconciles: 1, + }).SetupWithManager(mgr); err != nil { + log.Error(err, "Unable to create controller", "controller", "TTLController") + os.Exit(1) + } + log.Info("TTLController registered") + + // Add health checks + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + log.Error(err, "Unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + log.Error(err, "Unable to set up ready check") + os.Exit(1) + } + + log.Info("All controllers registered, starting manager") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + log.Error(err, "Problem running manager") + os.Exit(1) + } +} diff --git a/cmd/device-api-server/main.go b/cmd/device-api-server/main.go new file mode 100644 index 000000000..fe363a76f --- /dev/null +++ b/cmd/device-api-server/main.go @@ -0,0 +1,169 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package main implements the Device API Server. +// +// The Device API Server is a node-local gRPC cache server deployed as a +// Kubernetes DaemonSet. It acts as an intermediary between providers +// (health monitors) that update GPU device states and consumers +// (device plugins, DRA drivers) that read device states. +// +// Key features: +// - Read-blocking semantics: Reads are blocked during provider updates +// to prevent consumers from reading stale data +// - Multiple provider support: Multiple health monitors can update +// different conditions on the same GPUs +// - Multiple consumer support: Device plugins, DRA drivers, and other +// consumers can read and watch GPU states +// - Observability: Prometheus metrics, structured logging with klog/v2 +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "log/slog" + "os" + "os/signal" + "syscall" + "time" + + "k8s.io/klog/v2" + + "github.com/nvidia/nvsentinel/pkg/deviceapiserver" + "github.com/nvidia/nvsentinel/pkg/version" +) + +const ( + // ComponentName is the name of this component for logging. + ComponentName = "device-api-server" +) + +func main() { + // Create config with defaults + config := deviceapiserver.DefaultConfig() + + // Initialize klog flags first + klog.InitFlags(nil) + + // Bind our flags + showVersion := flag.Bool("version", false, "Show version and exit") + + // Manual duration flags (since BindFlags can't handle them properly) + var shutdownTimeout, shutdownDelay int + flag.StringVar(&config.GRPCAddress, "grpc-address", config.GRPCAddress, + "TCP address for gRPC server (e.g., :50051)") + flag.StringVar(&config.UnixSocket, "unix-socket", config.UnixSocket, + "Path to Unix socket for node-local IPC (empty to disable)") + flag.IntVar(&config.HealthPort, "health-port", config.HealthPort, + "Port for HTTP health endpoints (/healthz, /readyz)") + flag.IntVar(&config.MetricsPort, "metrics-port", config.MetricsPort, + "Port for Prometheus metrics (/metrics)") + flag.IntVar(&shutdownTimeout, "shutdown-timeout", int(config.ShutdownTimeout.Seconds()), + "Maximum time in seconds to wait for graceful shutdown") + flag.IntVar(&shutdownDelay, "shutdown-delay", int(config.ShutdownDelay.Seconds()), + "Time in seconds to wait before starting shutdown (for k8s readiness propagation)") + flag.StringVar(&config.LogFormat, "log-format", config.LogFormat, + "Log output format: text or json") + flag.StringVar(&config.NodeName, "node-name", config.NodeName, + "Kubernetes node name (defaults to NODE_NAME env var)") + + // NVML provider flags + flag.BoolVar(&config.NVMLEnabled, "enable-nvml-provider", config.NVMLEnabled, + "Enable the built-in NVML provider for device enumeration and health monitoring") + flag.StringVar(&config.NVMLDriverRoot, "nvml-driver-root", config.NVMLDriverRoot, + "Root path where NVIDIA driver libraries are located") + flag.StringVar(&config.NVMLIgnoredXids, "nvml-ignored-xids", config.NVMLIgnoredXids, + "Comma-separated list of additional XID error codes to ignore") + flag.BoolVar(&config.NVMLHealthCheckEnabled, "nvml-health-check", config.NVMLHealthCheckEnabled, + "Enable XID event monitoring for health checks") + + flag.Parse() + + // Apply duration conversions + config.ShutdownTimeout = time.Duration(shutdownTimeout) * time.Second + config.ShutdownDelay = time.Duration(shutdownDelay) * time.Second + + // Apply environment overrides + config.ApplyEnvironment() + + // Handle version flag + if *showVersion { + v := version.Get() + if config.LogFormat == "json" { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + if err := enc.Encode(v); err != nil { + fmt.Fprintf(os.Stderr, "Failed to encode version: %v\n", err) + os.Exit(1) + } + } else { + fmt.Println(v.String()) + } + os.Exit(0) + } + + // Validate configuration + if err := config.Validate(); err != nil { + fmt.Fprintf(os.Stderr, "Invalid configuration: %v\n", err) + os.Exit(1) + } + + // Configure klog for JSON output if requested + if config.LogFormat == "json" { + klog.SetSlogLogger(slog.New(slog.NewJSONHandler(os.Stdout, nil))) + } + + // Create root logger with component name + logger := klog.Background().WithName(ComponentName) + if config.NodeName != "" { + logger = logger.WithValues("node", config.NodeName) + } + + ctx := klog.NewContext(context.Background(), logger) + + // Log startup + versionInfo := version.Get() + logger.Info("Starting server", + "version", versionInfo.Version, + "commit", versionInfo.GitCommit, + "buildDate", versionInfo.BuildDate, + "config", map[string]interface{}{ + "grpcAddress": config.GRPCAddress, + "unixSocket": config.UnixSocket, + "healthPort": config.HealthPort, + "metricsPort": config.MetricsPort, + "shutdownTimeout": config.ShutdownTimeout.String(), + "shutdownDelay": config.ShutdownDelay.String(), + "logFormat": config.LogFormat, + "nvmlEnabled": config.NVMLEnabled, + "nvmlDriverRoot": config.NVMLDriverRoot, + "nvmlHealthCheck": config.NVMLHealthCheckEnabled, + }, + ) + + // Set up signal handling for graceful shutdown + ctx, cancel := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + // Create and start server + server := deviceapiserver.New(config, logger) + if err := server.Start(ctx); err != nil { + logger.Error(err, "Server error") + os.Exit(1) + } + + logger.Info("Server stopped gracefully") +} diff --git a/cmd/health-provider/main.go b/cmd/health-provider/main.go new file mode 100644 index 000000000..e6721b65c --- /dev/null +++ b/cmd/health-provider/main.go @@ -0,0 +1,129 @@ +//go:build nvml + +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Command health-provider is a standalone GPU health monitoring service. +// +// It monitors GPU health using NVML (and optionally other sources like syslog) +// and reports health events to the device-api-server via gRPC. +// +// This service is designed to run as a DaemonSet alongside device-api-server, +// enabling the three-tier architecture: +// +// HealthProvider-service → Device-Api-server → HealthEvent CRDs → Controllers +// +// Usage: +// +// health-provider \ +// --server-address=localhost:9001 \ +// --driver-root=/run/nvidia/driver \ +// --enable-nvml=true +package main + +import ( + "context" + "flag" + "os" + "os/signal" + "syscall" + + "k8s.io/klog/v2" + + "github.com/nvidia/nvsentinel/pkg/healthprovider" +) + +func main() { + // Initialize logging + klog.InitFlags(nil) + + cfg, nvmlCfg := parseFlags() + // flag.Parse() removed - already called in parseFlags() + + logger := klog.Background() + logger.Info("Starting health-provider", + "serverAddress", cfg.ServerAddress, + "providerID", cfg.ProviderID, + "nodeName", cfg.NodeName, + ) + + // Set up context with signal handling + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + go func() { + sig := <-sigCh + logger.Info("Received signal, shutting down", "signal", sig) + cancel() + }() + + // Create provider + provider := healthprovider.New(cfg, logger) + + // Register NVML source + if nvmlCfg.HealthCheckEnabled { + nvmlSource := healthprovider.NewNVMLSource(nvmlCfg, logger) + provider.RegisterSource(nvmlSource) + } + + // Run provider (blocks until context is cancelled) + if err := provider.Run(ctx); err != nil { + logger.Error(err, "Provider failed") + os.Exit(1) + } + + logger.Info("Health provider shutdown complete") +} + +func parseFlags() (healthprovider.Config, healthprovider.NVMLSourceConfig) { + cfg := healthprovider.DefaultConfig() + nvmlCfg := healthprovider.DefaultNVMLSourceConfig() + + // Provider flags + flag.StringVar(&cfg.ServerAddress, "server-address", cfg.ServerAddress, + "Address of device-api-server gRPC endpoint") + flag.StringVar(&cfg.ProviderID, "provider-id", cfg.ProviderID, + "Unique identifier for this provider instance") + flag.StringVar(&cfg.NodeName, "node-name", cfg.NodeName, + "Kubernetes node name (defaults to NODE_NAME env var)") + flag.IntVar(&cfg.HealthCheckPort, "health-port", cfg.HealthCheckPort, + "HTTP port for liveness/readiness probes") + + // NVML source flags + flag.StringVar(&nvmlCfg.DriverRoot, "driver-root", nvmlCfg.DriverRoot, + "Root path for NVIDIA driver libraries") + flag.BoolVar(&nvmlCfg.HealthCheckEnabled, "enable-nvml", nvmlCfg.HealthCheckEnabled, + "Enable NVML GPU health monitoring") + + // Parse flags first + flag.Parse() + + // Environment variable overrides + if addr := os.Getenv("PROVIDER_SERVER_ADDRESS"); addr != "" { + cfg.ServerAddress = addr + } + if id := os.Getenv("PROVIDER_ID"); id != "" { + cfg.ProviderID = id + } + if name := os.Getenv("NODE_NAME"); name != "" && cfg.NodeName == "" { + cfg.NodeName = name + } + if root := os.Getenv("NVML_DRIVER_ROOT"); root != "" { + nvmlCfg.DriverRoot = root + } + + return cfg, nvmlCfg +} diff --git a/cmd/health-provider/main_stub.go b/cmd/health-provider/main_stub.go new file mode 100644 index 000000000..3133ab975 --- /dev/null +++ b/cmd/health-provider/main_stub.go @@ -0,0 +1,103 @@ +//go:build !nvml + +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Command health-provider stub for building without NVML support. +package main + +import ( + "context" + "flag" + "os" + "os/signal" + "syscall" + + "k8s.io/klog/v2" + + "github.com/nvidia/nvsentinel/pkg/healthprovider" +) + +func main() { + klog.InitFlags(nil) + + cfg, nvmlCfg := parseFlags() + + logger := klog.Background() + logger.Info("Starting health-provider (stub build - no NVML support)", + "serverAddress", cfg.ServerAddress, + "providerID", cfg.ProviderID, + ) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + go func() { + sig := <-sigCh + logger.Info("Received signal, shutting down", "signal", sig) + cancel() + }() + + provider := healthprovider.New(cfg, logger) + + // Register stub NVML source (will fail to start but won't crash) + if nvmlCfg.HealthCheckEnabled { + nvmlSource := healthprovider.NewNVMLSource(nvmlCfg, logger) + provider.RegisterSource(nvmlSource) + } + + if err := provider.Run(ctx); err != nil { + logger.Error(err, "Provider failed") + os.Exit(1) + } + + logger.Info("Health provider shutdown complete") +} + +func parseFlags() (healthprovider.Config, healthprovider.NVMLSourceConfig) { + cfg := healthprovider.DefaultConfig() + nvmlCfg := healthprovider.DefaultNVMLSourceConfig() + + flag.StringVar(&cfg.ServerAddress, "server-address", cfg.ServerAddress, + "Address of device-api-server gRPC endpoint") + flag.StringVar(&cfg.ProviderID, "provider-id", cfg.ProviderID, + "Unique identifier for this provider instance") + flag.StringVar(&cfg.NodeName, "node-name", cfg.NodeName, + "Kubernetes node name") + flag.IntVar(&cfg.HealthCheckPort, "health-port", cfg.HealthCheckPort, + "HTTP port for health probes") + flag.StringVar(&nvmlCfg.DriverRoot, "driver-root", nvmlCfg.DriverRoot, + "Root path for NVIDIA driver libraries") + flag.BoolVar(&nvmlCfg.HealthCheckEnabled, "enable-nvml", nvmlCfg.HealthCheckEnabled, + "Enable NVML GPU health monitoring") + + flag.Parse() + + if addr := os.Getenv("PROVIDER_SERVER_ADDRESS"); addr != "" { + cfg.ServerAddress = addr + } + if id := os.Getenv("PROVIDER_ID"); id != "" { + cfg.ProviderID = id + } + if name := os.Getenv("NODE_NAME"); name != "" && cfg.NodeName == "" { + cfg.NodeName = name + } + if root := os.Getenv("NVML_DRIVER_ROOT"); root != "" { + nvmlCfg.DriverRoot = root + } + + return cfg, nvmlCfg +} diff --git a/cmd/nvml-provider/main.go b/cmd/nvml-provider/main.go new file mode 100644 index 000000000..c78a775f4 --- /dev/null +++ b/cmd/nvml-provider/main.go @@ -0,0 +1,716 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build nvml + +// Command nvml-provider is a standalone NVML-based GPU health provider that +// connects to a device-api-server instance via gRPC. +// +// This is designed to run as a sidecar container alongside device-api-server, +// providing GPU enumeration and health monitoring via NVML. +// +// Usage: +// +// nvml-provider --server-address=localhost:9001 --driver-root=/run/nvidia/driver +package main + +import ( + "context" + "flag" + "fmt" + "net/http" + "os" + "os/signal" + "path/filepath" + "sync" + "syscall" + "time" + + "github.com/NVIDIA/go-nvml/pkg/nvml" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/protobuf/types/known/timestamppb" + "k8s.io/klog/v2" + + v1alpha1 "github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1" +) + +const ( + // DefaultProviderID is the default identifier for this provider. + DefaultProviderID = "nvml-provider-sidecar" + + // ConditionTypeNVMLReady is the condition type for NVML health status. + ConditionTypeNVMLReady = "NVMLReady" + + // Condition status values. + ConditionStatusTrue = "True" + ConditionStatusFalse = "False" + ConditionStatusUnknown = "Unknown" + + // HeartbeatInterval is how often to send heartbeats. + HeartbeatInterval = 10 * time.Second + + // HealthCheckPort is the HTTP port for health checks. + HealthCheckPort = 8082 + + // EventTimeout is the timeout for NVML event wait (in milliseconds). + EventTimeout = 5000 + + // DefaultServerAddress is the default device-api-server address. + DefaultServerAddress = "localhost:9001" + + // ConnectionRetryInterval is how long to wait between connection attempts. + ConnectionRetryInterval = 5 * time.Second + + // MaxConnectionRetries is the maximum number of connection attempts. + MaxConnectionRetries = 60 +) + +// Config holds the provider configuration. +type Config struct { + ServerAddress string + ProviderID string + DriverRoot string + HealthCheckEnabled bool + HealthCheckPort int + IgnoredXids []uint64 +} + +// DefaultConfig returns a Config with sensible defaults. +func DefaultConfig() Config { + return Config{ + ServerAddress: DefaultServerAddress, + ProviderID: DefaultProviderID, + DriverRoot: "/run/nvidia/driver", + HealthCheckEnabled: true, + HealthCheckPort: HealthCheckPort, + } +} + +// Provider is the standalone NVML provider that connects to device-api-server. +type Provider struct { + config Config + logger klog.Logger + + // gRPC clients + conn *grpc.ClientConn + gpuClient v1alpha1.GpuServiceClient + healthClient grpc_health_v1.HealthClient + + // NVML + nvmllib nvml.Interface + eventSet nvml.EventSet + + // State + mu sync.RWMutex + gpuUUIDs []string + initialized bool + connected bool + healthy bool + monitorRunning bool + + // Lifecycle + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup +} + +// NewProvider creates a new standalone NVML provider. +func NewProvider(cfg Config, logger klog.Logger) *Provider { + return &Provider{ + config: cfg, + logger: logger.WithName("nvml-provider"), + } +} + +func main() { + // Initialize logging flags first + klog.InitFlags(nil) + + cfg := parseFlags() + // flag.Parse() is called inside parseFlags() + + logger := klog.Background() + logger.Info("Starting NVML provider sidecar", + "serverAddress", cfg.ServerAddress, + "providerID", cfg.ProviderID, + "driverRoot", cfg.DriverRoot, + "healthCheckEnabled", cfg.HealthCheckEnabled, + ) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Handle signals + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + go func() { + sig := <-sigCh + logger.Info("Received signal, shutting down", "signal", sig) + cancel() + }() + + // Create and run provider + provider := NewProvider(cfg, logger) + if err := provider.Run(ctx); err != nil { + logger.Error(err, "Provider failed") + os.Exit(1) + } + + logger.Info("NVML provider shutdown complete") +} + +func parseFlags() Config { + cfg := DefaultConfig() + + flag.StringVar(&cfg.ServerAddress, "server-address", cfg.ServerAddress, + "Address of device-api-server gRPC endpoint") + flag.StringVar(&cfg.ProviderID, "provider-id", cfg.ProviderID, + "Unique identifier for this provider") + flag.StringVar(&cfg.DriverRoot, "driver-root", cfg.DriverRoot, + "Root path for NVIDIA driver libraries") + flag.BoolVar(&cfg.HealthCheckEnabled, "health-check", cfg.HealthCheckEnabled, + "Enable XID event monitoring for health checks") + flag.IntVar(&cfg.HealthCheckPort, "health-port", cfg.HealthCheckPort, + "HTTP port for health check endpoints") + + // Parse flags + flag.Parse() + + // Check environment variables as fallback (override flags if set) + if addr := os.Getenv("PROVIDER_SERVER_ADDRESS"); addr != "" { + cfg.ServerAddress = addr + } + if id := os.Getenv("PROVIDER_ID"); id != "" { + cfg.ProviderID = id + } + if root := os.Getenv("NVML_DRIVER_ROOT"); root != "" { + cfg.DriverRoot = root + } + + return cfg +} + +// Run starts the provider and blocks until the context is cancelled. +func (p *Provider) Run(ctx context.Context) error { + p.ctx, p.cancel = context.WithCancel(ctx) + defer p.cancel() + + // Start health check server + p.wg.Add(1) + go p.runHealthServer() + + // Initialize NVML + if err := p.initNVML(); err != nil { + return fmt.Errorf("failed to initialize NVML: %w", err) + } + defer p.shutdownNVML() + + // Connect to server with retry + if err := p.connectWithRetry(); err != nil { + return fmt.Errorf("failed to connect to server: %w", err) + } + defer p.disconnect() + + // Enumerate and register GPUs (or reconcile if reconnecting) + if err := p.enumerateAndRegisterGPUs(); err != nil { + return fmt.Errorf("failed to enumerate GPUs: %w", err) + } + + // Reconcile state (handles restart/reconnection scenarios) + if err := p.ReconcileState(p.ctx); err != nil { + // Reconciliation failure is not fatal - log and continue + p.logger.Error(err, "State reconciliation failed, continuing") + } + + // Start heartbeat loop + p.wg.Add(1) + go p.runHeartbeatLoop() + + // Start health monitoring if enabled + if p.config.HealthCheckEnabled && len(p.gpuUUIDs) > 0 { + p.wg.Add(1) + go p.runHealthMonitor() + } + + // Mark as healthy + p.setHealthy(true) + + // Wait for shutdown + <-p.ctx.Done() + + // Graceful shutdown + p.setHealthy(false) + p.wg.Wait() + + return nil +} + +// initNVML initializes the NVML library. +func (p *Provider) initNVML() error { + // Find NVML library + libraryPath := p.findDriverLibrary() + if libraryPath != "" { + p.logger.V(2).Info("Using NVML library", "path", libraryPath) + p.nvmllib = nvml.New(nvml.WithLibraryPath(libraryPath)) + } else { + p.logger.V(2).Info("Using system default NVML library") + p.nvmllib = nvml.New() + } + + // Initialize + ret := p.nvmllib.Init() + if ret != nvml.SUCCESS { + return fmt.Errorf("NVML init failed: %v", nvml.ErrorString(ret)) + } + + // Log driver version + if version, ret := p.nvmllib.SystemGetDriverVersion(); ret == nvml.SUCCESS { + p.logger.Info("NVML initialized", "driverVersion", version) + } + + p.initialized = true + return nil +} + +// shutdownNVML shuts down the NVML library. +func (p *Provider) shutdownNVML() { + if !p.initialized { + return + } + + if p.eventSet != nil { + p.eventSet.Free() + p.eventSet = nil + } + + p.nvmllib.Shutdown() + p.initialized = false + p.logger.V(1).Info("NVML shutdown complete") +} + +// findDriverLibrary locates the NVML library in the driver root. +func (p *Provider) findDriverLibrary() string { + if p.config.DriverRoot == "" { + return "" + } + + paths := []string{ + filepath.Join(p.config.DriverRoot, "usr/lib64/libnvidia-ml.so.1"), + filepath.Join(p.config.DriverRoot, "usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1"), + filepath.Join(p.config.DriverRoot, "usr/lib/libnvidia-ml.so.1"), + filepath.Join(p.config.DriverRoot, "lib64/libnvidia-ml.so.1"), + filepath.Join(p.config.DriverRoot, "lib/libnvidia-ml.so.1"), + } + + for _, path := range paths { + if _, err := os.Stat(path); err == nil { + return path + } + } + + return "" +} + +// connectWithRetry connects to the device-api-server with retry logic. +func (p *Provider) connectWithRetry() error { + var lastErr error + + for i := 0; i < MaxConnectionRetries; i++ { + select { + case <-p.ctx.Done(): + return p.ctx.Err() + default: + } + + conn, err := grpc.NewClient( + p.config.ServerAddress, + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + lastErr = err + p.logger.V(1).Info("Connection attempt failed, retrying", + "attempt", i+1, + "error", err, + ) + time.Sleep(ConnectionRetryInterval) + continue + } + + p.conn = conn + p.gpuClient = v1alpha1.NewGpuServiceClient(conn) + p.healthClient = grpc_health_v1.NewHealthClient(conn) + + // Wait for server to be ready + if err := p.waitForServerReady(); err != nil { + conn.Close() + lastErr = err + p.logger.V(1).Info("Server not ready, retrying", + "attempt", i+1, + "error", err, + ) + time.Sleep(ConnectionRetryInterval) + continue + } + + p.connected = true + p.logger.Info("Connected to device-api-server", "address", p.config.ServerAddress) + return nil + } + + return fmt.Errorf("failed to connect after %d attempts: %w", MaxConnectionRetries, lastErr) +} + +// waitForServerReady waits for the server to report healthy. +func (p *Provider) waitForServerReady() error { + ctx, cancel := context.WithTimeout(p.ctx, 5*time.Second) + defer cancel() + + resp, err := p.healthClient.Check(ctx, &grpc_health_v1.HealthCheckRequest{}) + if err != nil { + return fmt.Errorf("health check failed: %w", err) + } + + if resp.Status != grpc_health_v1.HealthCheckResponse_SERVING { + return fmt.Errorf("server not serving: %v", resp.Status) + } + + return nil +} + +// disconnect closes the gRPC connection. +func (p *Provider) disconnect() { + if p.conn != nil { + p.conn.Close() + p.conn = nil + } + p.connected = false +} + +// enumerateAndRegisterGPUs discovers GPUs via NVML and registers them. +func (p *Provider) enumerateAndRegisterGPUs() error { + count, ret := p.nvmllib.DeviceGetCount() + if ret != nvml.SUCCESS { + return fmt.Errorf("failed to get device count: %v", nvml.ErrorString(ret)) + } + + if count == 0 { + p.logger.Info("No GPUs found on this node") + return nil + } + + p.logger.Info("Enumerating GPUs", "count", count) + p.gpuUUIDs = make([]string, 0, count) + + for i := 0; i < count; i++ { + device, ret := p.nvmllib.DeviceGetHandleByIndex(i) + if ret != nvml.SUCCESS { + p.logger.Error(nil, "Failed to get device handle", "index", i, "error", nvml.ErrorString(ret)) + continue + } + + uuid, ret := device.GetUUID() + if ret != nvml.SUCCESS { + p.logger.Error(nil, "Failed to get device UUID", "index", i, "error", nvml.ErrorString(ret)) + continue + } + + // Get device info for registration + productName, _ := device.GetName() + var memoryBytes uint64 + if memInfo, ret := device.GetMemoryInfo(); ret == nvml.SUCCESS { + memoryBytes = memInfo.Total + } + + // Register GPU with server + if err := p.registerGPU(uuid, productName, memoryBytes); err != nil { + p.logger.Error(err, "Failed to register GPU", "uuid", uuid) + continue + } + + p.gpuUUIDs = append(p.gpuUUIDs, uuid) + p.logger.Info("Registered GPU", + "uuid", uuid, + "productName", productName, + "memory", formatBytes(memoryBytes), + ) + } + + p.logger.Info("GPU enumeration complete", "registered", len(p.gpuUUIDs)) + return nil +} + +// registerGPU registers a single GPU with the device-api-server using CreateGpu. +func (p *Provider) registerGPU(uuid, productName string, memoryBytes uint64) error { + ctx, cancel := context.WithTimeout(p.ctx, 5*time.Second) + defer cancel() + + req := &v1alpha1.CreateGpuRequest{ + Gpu: &v1alpha1.Gpu{ + Metadata: &v1alpha1.ObjectMeta{Name: uuid}, + Spec: &v1alpha1.GpuSpec{Uuid: uuid}, + Status: &v1alpha1.GpuStatus{ + Conditions: []*v1alpha1.Condition{ + { + Type: ConditionTypeNVMLReady, + Status: ConditionStatusTrue, + Reason: "Initialized", + Message: fmt.Sprintf("GPU enumerated via NVML: %s (%s)", productName, formatBytes(memoryBytes)), + LastTransitionTime: timestamppb.Now(), + }, + }, + }, + }, + } + + _, err := p.gpuClient.CreateGpu(ctx, req) + return err +} + +// runHeartbeatLoop sends periodic heartbeats to the server. +func (p *Provider) runHeartbeatLoop() { + defer p.wg.Done() + + ticker := time.NewTicker(HeartbeatInterval) + defer ticker.Stop() + + for { + select { + case <-p.ctx.Done(): + return + case <-ticker.C: + if err := p.sendHeartbeat(); err != nil { + p.logger.Error(err, "Failed to send heartbeat") + } + } + } +} + +// sendHeartbeat performs a health check on the server connection. +// Note: The Heartbeat RPC was removed. We now just verify the server is reachable. +func (p *Provider) sendHeartbeat() error { + ctx, cancel := context.WithTimeout(p.ctx, 5*time.Second) + defer cancel() + + // Verify server connectivity by checking gRPC health + resp, err := p.healthClient.Check(ctx, &grpc_health_v1.HealthCheckRequest{}) + if err != nil { + return err + } + + if resp.Status != grpc_health_v1.HealthCheckResponse_SERVING { + return fmt.Errorf("server not serving: %v", resp.Status) + } + + p.mu.RLock() + gpuCount := len(p.gpuUUIDs) + p.mu.RUnlock() + + p.logger.V(4).Info("Health check passed", "gpuCount", gpuCount) + return nil +} + +// runHealthMonitor monitors NVML events for GPU health changes. +func (p *Provider) runHealthMonitor() { + defer p.wg.Done() + + p.mu.Lock() + p.monitorRunning = true + p.mu.Unlock() + + defer func() { + p.mu.Lock() + p.monitorRunning = false + p.mu.Unlock() + }() + + // Create event set + eventSet, ret := p.nvmllib.EventSetCreate() + if ret != nvml.SUCCESS { + p.logger.Error(nil, "Failed to create event set", "error", nvml.ErrorString(ret)) + return + } + p.eventSet = eventSet + + // Register devices for XID events + deviceCount, ret := p.nvmllib.DeviceGetCount() + if ret != nvml.SUCCESS { + p.logger.Error(nil, "Failed to get device count", "error", nvml.ErrorString(ret)) + return + } + + for i := 0; i < deviceCount; i++ { + device, ret := p.nvmllib.DeviceGetHandleByIndex(i) + if ret != nvml.SUCCESS { + continue + } + ret = device.RegisterEvents(nvml.EventTypeXidCriticalError|nvml.EventTypeSingleBitEccError|nvml.EventTypeDoubleBitEccError, eventSet) + if ret != nvml.SUCCESS { + p.logger.V(1).Info("Failed to register events for device", "index", i, "error", nvml.ErrorString(ret)) + } + } + + p.logger.Info("Health monitor started") + + // Event loop + for { + select { + case <-p.ctx.Done(): + return + default: + } + + data, ret := eventSet.Wait(EventTimeout) + if ret == nvml.ERROR_TIMEOUT { + continue + } + if ret != nvml.SUCCESS { + p.logger.V(1).Info("Event wait error", "error", nvml.ErrorString(ret)) + continue + } + + p.handleXIDEvent(data) + } +} + +// handleXIDEvent processes an XID error event. +func (p *Provider) handleXIDEvent(data nvml.EventData) { + uuid, ret := data.Device.GetUUID() + if ret != nvml.SUCCESS { + p.logger.Error(nil, "Failed to get device UUID from event") + return + } + + xid := data.EventData + p.logger.Info("XID event received", + "uuid", uuid, + "xid", xid, + "eventType", data.EventType, + ) + + // Update GPU status + status := ConditionStatusTrue + reason := "Healthy" + message := "GPU is healthy" + + if isCriticalXID(xid) { + status = ConditionStatusFalse + reason = "XIDError" + message = fmt.Sprintf("Critical XID error: %d", xid) + } + + ctx, cancel := context.WithTimeout(p.ctx, 5*time.Second) + defer cancel() + + req := &v1alpha1.UpdateGpuStatusRequest{ + Name: uuid, + Status: &v1alpha1.GpuStatus{ + Conditions: []*v1alpha1.Condition{ + { + Type: ConditionTypeNVMLReady, + Status: status, + Reason: reason, + Message: message, + LastTransitionTime: timestamppb.Now(), + }, + }, + }, + } + + if _, err := p.gpuClient.UpdateGpuStatus(ctx, req); err != nil { + p.logger.Error(err, "Failed to update GPU status", "uuid", uuid) + } +} + +// runHealthServer runs the HTTP health check server. +func (p *Provider) runHealthServer() { + defer p.wg.Done() + + mux := http.NewServeMux() + mux.HandleFunc("/healthz", p.handleHealthz) + mux.HandleFunc("/readyz", p.handleReadyz) + mux.HandleFunc("/livez", p.handleHealthz) + + server := &http.Server{ + Addr: fmt.Sprintf(":%d", p.config.HealthCheckPort), + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + } + + go func() { + <-p.ctx.Done() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + server.Shutdown(ctx) + }() + + p.logger.Info("Health server started", "port", p.config.HealthCheckPort) + if err := server.ListenAndServe(); err != http.ErrServerClosed { + p.logger.Error(err, "Health server error") + } +} + +func (p *Provider) handleHealthz(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("ok\n")) +} + +func (p *Provider) handleReadyz(w http.ResponseWriter, _ *http.Request) { + p.mu.RLock() + healthy := p.healthy + p.mu.RUnlock() + + if healthy { + w.WriteHeader(http.StatusOK) + w.Write([]byte("ok\n")) + } else { + w.WriteHeader(http.StatusServiceUnavailable) + w.Write([]byte("not ready\n")) + } +} + +func (p *Provider) setHealthy(healthy bool) { + p.mu.Lock() + p.healthy = healthy + p.mu.Unlock() +} + +// Critical XIDs that indicate hardware failure. +var criticalXIDs = map[uint64]bool{ + 48: true, 63: true, 64: true, 74: true, 79: true, 94: true, 95: true, 119: true, 120: true, +} + +func isCriticalXID(xid uint64) bool { + return criticalXIDs[xid] +} + +func formatBytes(bytes uint64) string { + const GB = 1024 * 1024 * 1024 + const MB = 1024 * 1024 + const KB = 1024 + + switch { + case bytes >= GB: + return fmt.Sprintf("%.1f GB", float64(bytes)/float64(GB)) + case bytes >= MB: + return fmt.Sprintf("%.1f MB", float64(bytes)/float64(MB)) + case bytes >= KB: + return fmt.Sprintf("%.1f KB", float64(bytes)/float64(KB)) + default: + return fmt.Sprintf("%d B", bytes) + } +} diff --git a/cmd/nvml-provider/reconciler.go b/cmd/nvml-provider/reconciler.go new file mode 100644 index 000000000..615cf70d1 --- /dev/null +++ b/cmd/nvml-provider/reconciler.go @@ -0,0 +1,302 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build nvml + +package main + +import ( + "context" + "fmt" + "time" + + "github.com/NVIDIA/go-nvml/pkg/nvml" + "google.golang.org/protobuf/types/known/timestamppb" + + v1alpha1 "github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1" +) + +// ReconcileState reconciles the provider's state with the device-api-server. +// +// This is called on startup and after reconnection to ensure: +// 1. GPUs that were removed while disconnected are unregistered +// 2. GPUs that were added while disconnected are registered +// 3. GPU health states are reconciled with current NVML state +// +// This handles scenarios like: +// - Provider crash and restart +// - Network partition recovery +// - GPU hotplug/removal during provider downtime +func (p *Provider) ReconcileState(ctx context.Context) error { + p.logger.Info("Starting state reconciliation") + + // Step 1: Get current state from server + cachedGPUs, err := p.listCachedGPUs(ctx) + if err != nil { + return fmt.Errorf("failed to list cached GPUs: %w", err) + } + + p.logger.V(1).Info("Retrieved cached GPU state", "count", len(cachedGPUs)) + + // Step 2: Get current GPU UUIDs from NVML + currentUUIDs, err := p.getCurrentGPUUUIDs() + if err != nil { + return fmt.Errorf("failed to get current GPU UUIDs: %w", err) + } + + p.logger.V(1).Info("Current GPUs from NVML", "count", len(currentUUIDs)) + + // Build lookup maps + cachedUUIDSet := make(map[string]*v1alpha1.Gpu) + for _, gpu := range cachedGPUs { + cachedUUIDSet[gpu.GetSpec().GetUuid()] = gpu + } + + currentUUIDSet := make(map[string]bool) + for _, uuid := range currentUUIDs { + currentUUIDSet[uuid] = true + } + + // Step 3: Find and unregister removed GPUs + for uuid := range cachedUUIDSet { + if !currentUUIDSet[uuid] { + p.logger.Info("GPU was removed, unregistering", "uuid", uuid) + if err := p.unregisterGPU(ctx, uuid); err != nil { + p.logger.Error(err, "Failed to unregister removed GPU", "uuid", uuid) + // Continue with other GPUs + } + } + } + + // Step 4: Find and register new GPUs + for _, uuid := range currentUUIDs { + if _, exists := cachedUUIDSet[uuid]; !exists { + p.logger.Info("New GPU found, registering", "uuid", uuid) + if err := p.registerNewGPU(ctx, uuid); err != nil { + p.logger.Error(err, "Failed to register new GPU", "uuid", uuid) + // Continue with other GPUs + } + } + } + + // Step 5: Reconcile health state for existing GPUs + for _, uuid := range currentUUIDs { + if cachedGPU, exists := cachedUUIDSet[uuid]; exists { + if err := p.reconcileGPUHealth(ctx, uuid, cachedGPU); err != nil { + p.logger.Error(err, "Failed to reconcile GPU health", "uuid", uuid) + // Continue with other GPUs + } + } + } + + // Step 6: Update local GPU list + p.mu.Lock() + p.gpuUUIDs = currentUUIDs + p.mu.Unlock() + + p.logger.Info("State reconciliation complete", + "totalGPUs", len(currentUUIDs), + ) + + return nil +} + +// listCachedGPUs retrieves the list of GPUs from the server cache. +// +// Note: This lists ALL GPUs, not just those from this provider. +// TODO: Add provider_id filtering to ListGpus RPC for efficiency. +func (p *Provider) listCachedGPUs(ctx context.Context) ([]*v1alpha1.Gpu, error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + req := &v1alpha1.ListGpusRequest{} + resp, err := p.gpuClient.ListGpus(ctx, req) + if err != nil { + return nil, err + } + + // Filter to only GPUs that might belong to this provider + // For now, we assume all GPUs belong to us since we're the only provider + // A more robust solution would use provider_id filtering + return resp.GetGpuList().GetItems(), nil +} + +// getCurrentGPUUUIDs gets the list of GPU UUIDs currently visible to NVML. +func (p *Provider) getCurrentGPUUUIDs() ([]string, error) { + count, ret := p.nvmllib.DeviceGetCount() + if ret != nvml.SUCCESS { + return nil, fmt.Errorf("failed to get device count: %v", nvml.ErrorString(ret)) + } + + uuids := make([]string, 0, count) + for i := 0; i < count; i++ { + device, ret := p.nvmllib.DeviceGetHandleByIndex(i) + if ret != nvml.SUCCESS { + continue + } + + uuid, ret := device.GetUUID() + if ret != nvml.SUCCESS { + continue + } + + uuids = append(uuids, uuid) + } + + return uuids, nil +} + +// unregisterGPU removes a GPU from the server using DeleteGpu. +func (p *Provider) unregisterGPU(ctx context.Context, uuid string) error { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + req := &v1alpha1.DeleteGpuRequest{ + Name: uuid, + } + + _, err := p.gpuClient.DeleteGpu(ctx, req) + return err +} + +// registerNewGPU registers a newly discovered GPU. +func (p *Provider) registerNewGPU(ctx context.Context, uuid string) error { + // Get device info from NVML + productName := "Unknown" + var memoryBytes uint64 + + // Find the device by UUID + count, ret := p.nvmllib.DeviceGetCount() + if ret == nvml.SUCCESS { + for i := 0; i < count; i++ { + device, ret := p.nvmllib.DeviceGetHandleByIndex(i) + if ret != nvml.SUCCESS { + continue + } + deviceUUID, ret := device.GetUUID() + if ret != nvml.SUCCESS || deviceUUID != uuid { + continue + } + + // Found the device + if name, ret := device.GetName(); ret == nvml.SUCCESS { + productName = name + } + if memInfo, ret := device.GetMemoryInfo(); ret == nvml.SUCCESS { + memoryBytes = memInfo.Total + } + break + } + } + + return p.registerGPU(uuid, productName, memoryBytes) +} + +// reconcileGPUHealth compares cached health state with current NVML state. +// +// If the GPU was marked as Unknown (due to provider timeout) but is now +// healthy per NVML, we update it back to healthy. +func (p *Provider) reconcileGPUHealth(ctx context.Context, uuid string, cachedGPU *v1alpha1.Gpu) error { + // Check if the cached state shows Unknown (from heartbeat timeout) + var cachedCondition *v1alpha1.Condition + for _, cond := range cachedGPU.GetStatus().GetConditions() { + if cond.GetType() == "Ready" || cond.GetType() == ConditionTypeNVMLReady { + cachedCondition = cond + break + } + } + + // If the condition is Unknown, query NVML and update if healthy + if cachedCondition != nil && cachedCondition.GetStatus() == ConditionStatusUnknown { + p.logger.Info("GPU has Unknown status, checking current NVML state", "uuid", uuid) + + // For now, if we can enumerate the GPU via NVML, consider it healthy + // A more sophisticated check would query specific health indicators + healthy, err := p.isGPUHealthy(uuid) + if err != nil { + return fmt.Errorf("failed to check GPU health: %w", err) + } + + if healthy { + p.logger.Info("GPU is healthy per NVML, updating status", "uuid", uuid) + return p.updateGPUCondition(ctx, uuid, ConditionStatusTrue, "Recovered", "GPU recovered after provider reconnection") + } + } + + return nil +} + +// isGPUHealthy checks if a GPU is healthy via NVML. +func (p *Provider) isGPUHealthy(uuid string) (bool, error) { + // Find device by UUID + count, ret := p.nvmllib.DeviceGetCount() + if ret != nvml.SUCCESS { + return false, fmt.Errorf("failed to get device count: %v", nvml.ErrorString(ret)) + } + + for i := 0; i < count; i++ { + device, ret := p.nvmllib.DeviceGetHandleByIndex(i) + if ret != nvml.SUCCESS { + continue + } + deviceUUID, ret := device.GetUUID() + if ret != nvml.SUCCESS || deviceUUID != uuid { + continue + } + + // Device found - check basic health indicators + // 1. Can we get memory info? (basic liveness check) + if _, ret := device.GetMemoryInfo(); ret != nvml.SUCCESS { + return false, nil + } + + // 2. Check for pending page retirements (ECC errors) + if pending, ret := device.GetRetiredPagesPendingStatus(); ret == nvml.SUCCESS { + if pending == nvml.FEATURE_ENABLED { + p.logger.V(1).Info("GPU has pending page retirements", "uuid", uuid) + return false, nil + } + } + + // Device is accessible and no pending issues + return true, nil + } + + // Device not found - not healthy + return false, nil +} + +// updateGPUCondition updates a GPU's status via UpdateGpuStatus. +func (p *Provider) updateGPUCondition(ctx context.Context, uuid, status, reason, message string) error { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + req := &v1alpha1.UpdateGpuStatusRequest{ + Name: uuid, + Status: &v1alpha1.GpuStatus{ + Conditions: []*v1alpha1.Condition{ + { + Type: ConditionTypeNVMLReady, + Status: status, + Reason: reason, + Message: message, + LastTransitionTime: timestamppb.Now(), + }, + }, + }, + } + + _, err := p.gpuClient.UpdateGpuStatus(ctx, req) + return err +} diff --git a/code-generator/CONFIGURATION.md b/code-generator/CONFIGURATION.md new file mode 100644 index 000000000..5229f69da --- /dev/null +++ b/code-generator/CONFIGURATION.md @@ -0,0 +1,35 @@ +# NVIDIA Device API: Code Generator Configuration + +The code generation pipeline relies on specific versions of upstream tools. The source of truth for these versions is the root [/.versions.yaml](../../.versions.yaml). + +--- + +## Version Precedence + +1. **Override**: Environment variables set in the shell. +2. **Default**: Values defined in `.versions.yaml`. +3. **Fallback**: Tool-specific defaults. + +--- + +## Overriding Versions + +You can override defaults by setting environment variables before running the generation pipeline (e.g., `make code-gen`). + +| Environment Variable | Tool Category | Key in `.versions.yaml` | +|---------------------------|------------------------------|-------------------------| +| `KUBE_CODEGEN_TAG` | Kubernetes Generators | `kubernetes_code_gen` | +| `PROTOC_GEN_GO_TAG` | Protobuf Go Plugin | `protoc_gen_go` | +| `PROTOC_GEN_GO_GRPC_TAG` | gRPC Go Plugin | `protoc_gen_go_grpc` | +| `GOVERTER_TAG` | Proto-to-Go Mapper | `goverter` | + +### Example: Testing a Different Kubernetes Version + +If you need to verify the generator against a different version of the Kubernetes upstream code-generator: + +```bash +export KUBE_CODEGEN_TAG="v0.32.0" +./hack/update-codegen.sh +``` + +--- diff --git a/code-generator/LICENSE b/code-generator/LICENSE new file mode 100644 index 000000000..f4f87bd4e --- /dev/null +++ b/code-generator/LICENSE @@ -0,0 +1,203 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + \ No newline at end of file diff --git a/code-generator/README.md b/code-generator/README.md new file mode 100644 index 000000000..15d03b634 --- /dev/null +++ b/code-generator/README.md @@ -0,0 +1,38 @@ +# NVIDIA Device API: Code Generator + +The `code-generator` module contains custom Golang code-generators used to implement [Kubernetes-style API types](https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md) backed by **gRPC transport**. + +These generators enable the creation of native, versioned Go clients that communicate over Unix Domain Sockets (UDS) instead of the standard Kubernetes REST API. + +--- + +## Structure + +* **Orchestration Logic**: `kube_codegen.sh` is a bash library that provides a functional interface for managing the full generation lifecycle (Protos, DeepCopy, Goverter, and Clients). +* **Customized Generator**: A modified version of `client-gen` (located in `cmd/client-gen`) that replaces standard REST/JSON templates with gRPC-specific logic for the NVIDIA Device API. + +--- + +## Usage + +This module is primarily orchestrated by the root `Makefile` via `make code-gen`. The `kube_codegen.sh` script is designed to be **sourced** by other build scripts, not executed directly. See [client-go/hack/update-codegen.sh](../client-go/hack/update-codegen.sh) for an implementation example. + +### Available Functions + +- `kube::codegen::gen_proto_bindings`: Scans for `.proto` files and generates Go bindings (`.pb.go`) and gRPC interfaces (`_grpc.pb.go`). +- `kube::codegen::gen_helpers`: Runs upstream generators (`deepcopy`, `defaulter`, `validation`, `conversion`, [Goverter](https://github.com/jmattheis/goverter)). +- `kube::codegen::gen_client`: Compiles the customized `client-gen` binary and executes it to produce the Kubernetes-style stack: **Clientset**, **Listers**, and **Informers**. + +--- + +## Configuration + +Tool versions are managed in the central [`.versions.yaml`](../.versions.yaml) file at the repository root. See the [Configuration Guide](CONFIGURATION.md) for details on version pinning and environment variable overrides. + +--- + +## Modifying the Generator + +If you need to change the behavior of the generated client (e.g., adding default gRPC timeout logic or custom interceptors), modify the Go templates found in `cmd/client-gen/generators`. Detailed instructions can be found in the [client-gen README](./cmd/client-gen/README.md). + +--- diff --git a/code-generator/cmd/client-gen/README.md b/code-generator/cmd/client-gen/README.md new file mode 100644 index 000000000..4191140e5 --- /dev/null +++ b/code-generator/cmd/client-gen/README.md @@ -0,0 +1,65 @@ +# NVIDIA Device API: Client Generator + +This is a modified version of the Kubernetes [client-gen](https://github.com/kubernetes/code-generator/tree/master/cmd/client-gen). + +It generates a typed, versioned Go **Clientset**. Unlike the standard generator which defaults to REST/HTTP, this version creates clients that use **gRPC transport** to communicate with the node-local NVIDIA Device API. + +--- + +## Workflow + +### 1. Tagging API Types + +In your API definition files (e.g., `api/device/v1alpha1/types.go`), mark the types (e.g., `GPU`) that you want to generate clients for using `// +genclient`: + +* `// +genclient` - Generate default client verb functions (`create`, `update`, `delete`, `get`, `list`, `patch`, `watch`, and `updateStatus`). +* `// +genclient:nonNamespaced` - Generate verb functions without namespace parameters. +* `// +genclient:onlyVerbs=,` - Generate **only** the listed verbs. +* `// +genclient:skipVerbs=,` - Generate all default verbs **except** the listed ones. +* `// +genclient:noStatus` - Skip `updateStatus` verb even if the `.Status` struct field exists. +* `// +groupName=policy.authorization.k8s.io` – Overrides the API group name (defaults to the package name). +* `// +groupGoName=AuthorizationPolicy` – Sets a custom Golang identifier to de-conflict groups (defaults to the upper-case first segment of the group name). + +### 2. Running the Generator + +> [!NOTE] +> This binary is typically invoked via [kube_codegen.sh](../../README.md). + +When invoked, the generator resolves packages by combining `--input-base` (Go types) and `--proto-base` (gRPC stubs) to create unified transport logic. + +### 3. Adding Expansion Methods + +`client-gen` generates standard methods. To add custom logic (e.g., specialized filters) to a client, create a file named `${TYPE}_expansion.go` in the generated directory. The generator will automatically detect and embed the `${TYPE}Expansion` interface into the client. + +--- + +## Output Structure + +The generator produces a tiered directory structure. By default, the layout follows this pattern: + +- **Clientset**: Found at `${output-dir}/${versioned-name}/${clientset-name}/`. The primary entry point for consumers. +- **Typed Clients**: Found at `${output-dir}/${versioned-name}/${clientset-name}/typed/${group}/${version}/`. Contains the actual gRPC-backed implementation for each resource type. +- **Fake Clientset**: Found at `${output-dir}/${versioned-name}/${clientset-name}/fake/`. Used for unit testing without a running gRPC server. +- **Listers**: Found at `${output-dir}/listers/`. Provides a read-only, cached view of resources for high-speed lookups. +- **Informers**: Found at `${output-dir}/informers/`. Keeps the local cache updated via gRPC watch streams and provides event triggers for controllers. + +--- + +## Flags + +| Flag | Required | Description | +|------|----------|-------------| +| `--proto-base` | **Yes** | The base Go import path for the generated Protobuf stubs (e.g., `github.com/org/repo/api/gen/go`). | +| `--input` | **Yes** | Comma-separated list of groups/versions to generate (e.g., `device/v1alpha1,networking/v1`). | +| `--input-base` | **Yes** | Base import path for the API types (e.g., `github.com/org/repo/api`). | +| `--output-pkg` | **Yes** | Go package path for the generated files (e.g., `github.com/org/repo/client`). | +| `--output-dir` | **Yes** | Base directory for the output on disk (e.g., `./client`). | +| `--boilerplate` | No | Path to a header file (copyright/license) to prepend to generated files. Default: `hack/boilerplate.go.txt`. | +| `--clientset-name` | No | Name of the generated package/directory. Default: `clientset`. | +| `--versioned-name` | No | Name of the versioned clientset directory. Default: `versioned`. | +| `--plural-exceptions`| No | Comma-separated list of `Type:PluralizedType` overrides. | +| `--fake-clientset` | No | Generate a fake clientset that can be used in tests. Default: `true` | +| `--prefers-protobuf` | **N/A** | **Removed** This generator assumes Protobuf/gRPC support is always enabled. | +| `--apply-configuration-package` | **N/A** | **Not Implemented** | + +--- diff --git a/code-generator/cmd/client-gen/args/args.go b/code-generator/cmd/client-gen/args/args.go new file mode 100644 index 000000000..570349d9a --- /dev/null +++ b/code-generator/cmd/client-gen/args/args.go @@ -0,0 +1,148 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Portions Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved. + +Modified from the original to support gRPC transport. +Origin: https://github.com/kubernetes/code-generator/blob/v0.34.1/cmd/client-gen/args/args.go +*/ + +package args + +import ( + "fmt" + + "github.com/spf13/pflag" + + "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/types" +) + +type Args struct { + // The directory for the generated results. + OutputDir string + + // The Go import-path of the generated results. + OutputPkg string + + // The boilerplate header for Go files. + GoHeaderFile string + + // A sorted list of group versions to generate. For each of them the package path is found + // in GroupVersionToInputPath. + Groups []types.GroupVersions + + // Overrides for which types should be included in the client. + IncludedTypesOverrides map[types.GroupVersion][]string + + // ClientsetName is the name of the clientset to be generated. It's + // populated from command-line arguments. + ClientsetName string + // ClientsetAPIPath is the default API HTTP path for generated clients. + ClientsetAPIPath string + // ClientsetOnly determines if we should generate the clients for groups and + // types along with the clientset. It's populated from command-line + // arguments. + ClientsetOnly bool + // FakeClient determines if client-gen generates the fake clients. + FakeClient bool + // PluralExceptions specify list of exceptions used when pluralizing certain types. + // For example 'Endpoints:Endpoints', otherwise the pluralizer will generate 'Endpointes'. + PluralExceptions []string + + // ProtoBase is the base Go import-path of the protobuf stubs. + ProtoBase string +} + +func New() *Args { + return &Args{ + ClientsetName: "internalclientset", + ClientsetAPIPath: "/apis", + ClientsetOnly: false, + FakeClient: true, + } +} + +func (a *Args) AddFlags(fs *pflag.FlagSet, inputBase string) { + gvsBuilder := NewGroupVersionsBuilder(&a.Groups) + fs.StringVar(&a.OutputDir, "output-dir", "", + "the base directory under which to generate results") + fs.StringVar(&a.OutputPkg, "output-pkg", a.OutputPkg, + "the Go import-path of the generated results") + fs.StringVar(&a.GoHeaderFile, "go-header-file", "", + "the path to a file containing boilerplate header text; the string \"YEAR\" will be replaced with the current 4-digit year") + fs.Var(NewGVPackagesValue(gvsBuilder, nil), "input", + `group/versions that client-gen will generate clients for. At most one version per group is allowed. Specified in the format "group1/version1,group2/version2...".`) + fs.Var(NewGVTypesValue(&a.IncludedTypesOverrides, []string{}), "included-types-overrides", + "list of group/version/type for which client should be generated. By default, client is generated for all types which have genclient in types.go. This overrides that. For each groupVersion in this list, only the types mentioned here will be included. The default check of genclient will be used for other group versions.") + fs.Var(NewInputBasePathValue(gvsBuilder, inputBase), "input-base", + "base path to look for the api group.") + fs.StringVarP(&a.ClientsetName, "clientset-name", "n", a.ClientsetName, + "the name of the generated clientset package.") + fs.StringVarP(&a.ClientsetAPIPath, "clientset-api-path", "", a.ClientsetAPIPath, + "the value of default API HTTP path, starting with / and without trailing /.") + fs.BoolVar(&a.ClientsetOnly, "clientset-only", a.ClientsetOnly, + "when set, client-gen only generates the clientset shell, without generating the individual typed clients") + fs.BoolVar(&a.FakeClient, "fake-clientset", a.FakeClient, + "when set, client-gen will generate the fake clientset that can be used in tests") + fs.StringSliceVar(&a.PluralExceptions, "plural-exceptions", a.PluralExceptions, + "list of comma separated plural exception definitions in Type:PluralizedType form") + fs.StringVar(&a.ProtoBase, "proto-base", "", + "the base Go import-path of the protobuf stubs") + + // support old flags + fs.SetNormalizeFunc(mapFlagName("clientset-path", "output-pkg", fs.GetNormalizeFunc())) +} + +func (a *Args) Validate() error { + if len(a.OutputDir) == 0 { + return fmt.Errorf("--output-dir must be specified") + } + if len(a.OutputPkg) == 0 { + return fmt.Errorf("--output-pkg must be specified") + } + if len(a.ClientsetName) == 0 { + return fmt.Errorf("--clientset-name must be specified") + } + if len(a.ClientsetAPIPath) == 0 { + return fmt.Errorf("--clientset-api-path cannot be empty") + } + if len(a.ProtoBase) == 0 { + return fmt.Errorf("--proto-base must be specified") + } + + return nil +} + +// GroupVersionPackages returns a map from GroupVersion to the package with the types.go. +func (a *Args) GroupVersionPackages() map[types.GroupVersion]string { + res := map[types.GroupVersion]string{} + for _, pkg := range a.Groups { + for _, v := range pkg.Versions { + res[types.GroupVersion{Group: pkg.Group, Version: v.Version}] = v.Package + } + } + return res +} + +func mapFlagName(from, to string, old func(fs *pflag.FlagSet, name string) pflag.NormalizedName) func(fs *pflag.FlagSet, name string) pflag.NormalizedName { + return func(fs *pflag.FlagSet, name string) pflag.NormalizedName { + if name == from { + name = to + } + return old(fs, name) + } +} diff --git a/code-generator/cmd/client-gen/args/gvpackages.go b/code-generator/cmd/client-gen/args/gvpackages.go new file mode 100644 index 000000000..64331d4c0 --- /dev/null +++ b/code-generator/cmd/client-gen/args/gvpackages.go @@ -0,0 +1,182 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Portions Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved. + +Modified from the original to support gRPC transport. +Origin: https://github.com/kubernetes/code-generator/blob/v0.34.1/cmd/client-gen/args/gvpackages.go +*/ + +package args + +import ( + "bytes" + "encoding/csv" + "flag" + "path" + "sort" + "strings" + + "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/generators/util" + "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/types" +) + +type inputBasePathValue struct { + builder *groupVersionsBuilder +} + +var _ flag.Value = &inputBasePathValue{} + +func NewInputBasePathValue(builder *groupVersionsBuilder, def string) *inputBasePathValue { + v := &inputBasePathValue{ + builder: builder, + } + v.Set(def) + return v +} + +func (s *inputBasePathValue) Set(val string) error { + s.builder.importBasePath = val + return s.builder.update() +} + +func (s *inputBasePathValue) Type() string { + return "string" +} + +func (s *inputBasePathValue) String() string { + return s.builder.importBasePath +} + +type gvPackagesValue struct { + builder *groupVersionsBuilder + groups []string + changed bool +} + +func NewGVPackagesValue(builder *groupVersionsBuilder, def []string) *gvPackagesValue { + gvp := new(gvPackagesValue) + gvp.builder = builder + if def != nil { + if err := gvp.set(def); err != nil { + panic(err) + } + } + return gvp +} + +var _ flag.Value = &gvPackagesValue{} + +func (s *gvPackagesValue) set(vs []string) error { + if s.changed { + s.groups = append(s.groups, vs...) + } else { + s.groups = append([]string(nil), vs...) + } + + s.builder.groups = s.groups + return s.builder.update() +} + +func (s *gvPackagesValue) Set(val string) error { + vs, err := readAsCSV(val) + if err != nil { + return err + } + if err := s.set(vs); err != nil { + return err + } + s.changed = true + return nil +} + +func (s *gvPackagesValue) Type() string { + return "stringSlice" +} + +func (s *gvPackagesValue) String() string { + str, _ := writeAsCSV(s.groups) + return "[" + str + "]" +} + +type groupVersionsBuilder struct { + value *[]types.GroupVersions + groups []string + importBasePath string +} + +func NewGroupVersionsBuilder(groups *[]types.GroupVersions) *groupVersionsBuilder { + return &groupVersionsBuilder{ + value: groups, + } +} + +func (p *groupVersionsBuilder) update() error { + var seenGroups = make(map[types.Group]*types.GroupVersions) + for _, v := range p.groups { + pth, gvString := util.ParsePathGroupVersion(v) + gv, err := types.ToGroupVersion(gvString) + if err != nil { + return err + } + + versionPkg := types.PackageVersion{Package: path.Join(p.importBasePath, pth, gv.Group.NonEmpty(), gv.Version.String()), Version: gv.Version} + if group, ok := seenGroups[gv.Group]; ok { + vers := group.Versions + vers = append(vers, versionPkg) + seenGroups[gv.Group].Versions = vers + } else { + seenGroups[gv.Group] = &types.GroupVersions{ + PackageName: gv.Group.NonEmpty(), + Group: gv.Group, + Versions: []types.PackageVersion{versionPkg}, + } + } + } + + var groupNames []string + for groupName := range seenGroups { + groupNames = append(groupNames, groupName.String()) + } + sort.Strings(groupNames) + *p.value = []types.GroupVersions{} + for _, groupName := range groupNames { + *p.value = append(*p.value, *seenGroups[types.Group(groupName)]) + } + + return nil +} + +func readAsCSV(val string) ([]string, error) { + if val == "" { + return []string{}, nil + } + stringReader := strings.NewReader(val) + csvReader := csv.NewReader(stringReader) + return csvReader.Read() +} + +func writeAsCSV(vals []string) (string, error) { + b := &bytes.Buffer{} + w := csv.NewWriter(b) + err := w.Write(vals) + if err != nil { + return "", err + } + w.Flush() + return strings.TrimSuffix(b.String(), "\n"), nil +} diff --git a/code-generator/cmd/client-gen/args/gvpackages_test.go b/code-generator/cmd/client-gen/args/gvpackages_test.go new file mode 100644 index 000000000..5bceefb04 --- /dev/null +++ b/code-generator/cmd/client-gen/args/gvpackages_test.go @@ -0,0 +1,123 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Portions Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved. + +Modified from the original to support gRPC transport. +Origin: https://github.com/kubernetes/code-generator/blob/v0.34.1/cmd/client-gen/args/gvpackages_test.go +*/ + +package args + +import ( + "fmt" + "reflect" + "strings" + "testing" + + "github.com/spf13/pflag" + + "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/types" +) + +func TestGVPackageFlag(t *testing.T) { + tests := []struct { + args []string + def []string + importBasePath string + expected map[types.GroupVersion]string + expectedGroups []types.GroupVersions + parseError string + }{ + { + args: []string{}, + expected: map[types.GroupVersion]string{}, + expectedGroups: []types.GroupVersions{}, + }, + { + args: []string{"foo/bar/v1", "foo/bar/v2", "foo/bar/", "foo/v1"}, + expectedGroups: []types.GroupVersions{ + {PackageName: "bar", Group: types.Group("bar"), Versions: []types.PackageVersion{ + {Version: "v1", Package: "foo/bar/v1"}, + {Version: "v2", Package: "foo/bar/v2"}, + {Version: "", Package: "foo/bar"}, + }}, + {PackageName: "foo", Group: types.Group("foo"), Versions: []types.PackageVersion{ + {Version: "v1", Package: "foo/v1"}, + }}, + }, + }, + { + args: []string{"foo/bar/v1", "foo/bar/v2", "foo/bar/", "foo/v1"}, + def: []string{"foo/bar/v1alpha1", "foo/v1"}, + expectedGroups: []types.GroupVersions{ + {PackageName: "bar", Group: types.Group("bar"), Versions: []types.PackageVersion{ + {Version: "v1", Package: "foo/bar/v1"}, + {Version: "v2", Package: "foo/bar/v2"}, + {Version: "", Package: "foo/bar"}, + }}, + {PackageName: "foo", Group: types.Group("foo"), Versions: []types.PackageVersion{ + {Version: "v1", Package: "foo/v1"}, + }}, + }, + }, + { + args: []string{"api/v1", "api"}, + expectedGroups: []types.GroupVersions{ + {PackageName: "api", Group: types.Group("api"), Versions: []types.PackageVersion{ + {Version: "v1", Package: "api/v1"}, + {Version: "", Package: "api"}, + }}, + }, + }, + { + args: []string{"foo/v1"}, + importBasePath: "k8s.io/api", + expectedGroups: []types.GroupVersions{ + {PackageName: "foo", Group: types.Group("foo"), Versions: []types.PackageVersion{ + {Version: "v1", Package: "k8s.io/api/foo/v1"}, + }}, + }, + }, + } + for i, test := range tests { + fs := pflag.NewFlagSet("testGVPackage", pflag.ContinueOnError) + groups := []types.GroupVersions{} + builder := NewGroupVersionsBuilder(&groups) + fs.Var(NewGVPackagesValue(builder, test.def), "input", "usage") + fs.Var(NewInputBasePathValue(builder, test.importBasePath), "input-base-path", "usage") + + args := []string{} + for _, a := range test.args { + args = append(args, fmt.Sprintf("--input=%s", a)) + } + + err := fs.Parse(args) + if test.parseError != "" { + if err == nil { + t.Errorf("%d: expected error %q, got nil", i, test.parseError) + } else if !strings.Contains(err.Error(), test.parseError) { + t.Errorf("%d: expected error %q, got %q", i, test.parseError, err) + } + } else if err != nil { + t.Errorf("%d: expected nil error, got %v", i, err) + } + if !reflect.DeepEqual(groups, test.expectedGroups) { + t.Errorf("%d: expected groups %+v, got groups %+v", i, test.expectedGroups, groups) + } + } +} diff --git a/code-generator/cmd/client-gen/args/gvtype.go b/code-generator/cmd/client-gen/args/gvtype.go new file mode 100644 index 000000000..90a8e3373 --- /dev/null +++ b/code-generator/cmd/client-gen/args/gvtype.go @@ -0,0 +1,117 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Portions Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved. + +Modified from the original to support gRPC transport. +Origin: https://github.com/kubernetes/code-generator/blob/v0.34.1/cmd/client-gen/args/gvtype.go +*/ + +package args + +import ( + "flag" + "fmt" + "strings" + + "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/types" +) + +type gvTypeValue struct { + gvToTypes *map[types.GroupVersion][]string + changed bool +} + +func NewGVTypesValue(gvToTypes *map[types.GroupVersion][]string, def []string) *gvTypeValue { + gvt := new(gvTypeValue) + gvt.gvToTypes = gvToTypes + if def != nil { + if err := gvt.set(def); err != nil { + panic(err) + } + } + return gvt +} + +var _ flag.Value = &gvTypeValue{} + +func (s *gvTypeValue) set(vs []string) error { + if !s.changed { + *s.gvToTypes = map[types.GroupVersion][]string{} + } + + for _, input := range vs { + gvString, typeStr, err := parseGroupVersionType(input) + if err != nil { + return err + } + gv, err := types.ToGroupVersion(gvString) + if err != nil { + return err + } + types, ok := (*s.gvToTypes)[gv] + if !ok { + types = []string{} + } + types = append(types, typeStr) + (*s.gvToTypes)[gv] = types + } + + return nil +} + +func (s *gvTypeValue) Set(val string) error { + vs, err := readAsCSV(val) + if err != nil { + return err + } + if err := s.set(vs); err != nil { + return err + } + s.changed = true + return nil +} + +func (s *gvTypeValue) Type() string { + return "stringSlice" +} + +func (s *gvTypeValue) String() string { + strs := make([]string, 0, len(*s.gvToTypes)) + for gv, ts := range *s.gvToTypes { + for _, t := range ts { + strs = append(strs, gv.Group.String()+"/"+gv.Version.String()+"/"+t) + } + } + str, _ := writeAsCSV(strs) + return "[" + str + "]" +} + +func parseGroupVersionType(gvtString string) (gvString string, typeStr string, err error) { + invalidFormatErr := fmt.Errorf("invalid value: %s, should be of the form group/version/type", gvtString) + subs := strings.Split(gvtString, "/") + length := len(subs) + switch length { + case 2: + // gvtString of the form group/type, e.g. api/Service,extensions/ReplicaSet + return subs[0] + "/", subs[1], nil + case 3: + return strings.Join(subs[:length-1], "/"), subs[length-1], nil + default: + return "", "", invalidFormatErr + } +} diff --git a/code-generator/cmd/client-gen/generators/client_generator.go b/code-generator/cmd/client-gen/generators/client_generator.go new file mode 100644 index 000000000..cc7ff0fdb --- /dev/null +++ b/code-generator/cmd/client-gen/generators/client_generator.go @@ -0,0 +1,459 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Portions Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved. + +Modified from the original to support gRPC transport. +Origin: https://github.com/kubernetes/code-generator/blob/v0.34.1/cmd/client-gen/generators/client_generator.go +*/ + +// Package generators has the generators for the client-gen utility. +package generators + +import ( + "fmt" + "path" + "path/filepath" + "strings" + + "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/args" + "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/generators/fake" + "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/generators/scheme" + "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/generators/util" + clientgentypes "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/types" + + codegennamer "k8s.io/code-generator/pkg/namer" + genutil "k8s.io/code-generator/pkg/util" + "k8s.io/gengo/v2" + "k8s.io/gengo/v2/generator" + "k8s.io/gengo/v2/namer" + "k8s.io/gengo/v2/types" + + "k8s.io/klog/v2" +) + +// NameSystems returns the name system used by the generators in this package. +func NameSystems(pluralExceptions map[string]string) namer.NameSystems { + lowercaseNamer := namer.NewAllLowercasePluralNamer(pluralExceptions) + + publicNamer := &ExceptionNamer{ + Exceptions: map[string]string{ + // these exceptions are used to deconflict the generated code + // you can put your fully qualified package like + // to generate a name that doesn't conflict with your group. + // "k8s.io/apis/events/v1beta1.Event": "EventResource" + }, + KeyFunc: func(t *types.Type) string { + return t.Name.Package + "." + t.Name.Name + }, + Delegate: namer.NewPublicNamer(0), + } + privateNamer := &ExceptionNamer{ + Exceptions: map[string]string{ + // these exceptions are used to deconflict the generated code + // you can put your fully qualified package like + // to generate a name that doesn't conflict with your group. + // "k8s.io/apis/events/v1beta1.Event": "eventResource" + }, + KeyFunc: func(t *types.Type) string { + return t.Name.Package + "." + t.Name.Name + }, + Delegate: namer.NewPrivateNamer(0), + } + publicPluralNamer := &ExceptionNamer{ + Exceptions: map[string]string{ + // these exceptions are used to deconflict the generated code + // you can put your fully qualified package like + // to generate a name that doesn't conflict with your group. + // "k8s.io/apis/events/v1beta1.Event": "EventResource" + }, + KeyFunc: func(t *types.Type) string { + return t.Name.Package + "." + t.Name.Name + }, + Delegate: namer.NewPublicPluralNamer(pluralExceptions), + } + privatePluralNamer := &ExceptionNamer{ + Exceptions: map[string]string{ + // you can put your fully qualified package like + // to generate a name that doesn't conflict with your group. + // "k8s.io/apis/events/v1beta1.Event": "eventResource" + // these exceptions are used to deconflict the generated code + "k8s.io/apis/events/v1beta1.Event": "eventResources", + "k8s.io/kubernetes/pkg/apis/events.Event": "eventResources", + }, + KeyFunc: func(t *types.Type) string { + return t.Name.Package + "." + t.Name.Name + }, + Delegate: namer.NewPrivatePluralNamer(pluralExceptions), + } + + return namer.NameSystems{ + "singularKind": namer.NewPublicNamer(0), + "public": publicNamer, + "private": privateNamer, + "raw": namer.NewRawNamer("", nil), + "publicPlural": publicPluralNamer, + "privatePlural": privatePluralNamer, + "allLowercasePlural": lowercaseNamer, + "resource": codegennamer.NewTagOverrideNamer("resourceName", lowercaseNamer), + } +} + +// ExceptionNamer allows you specify exceptional cases with exact names. This allows you to have control +// for handling various conflicts, like group and resource names for instance. +type ExceptionNamer struct { + Exceptions map[string]string + KeyFunc func(*types.Type) string + + Delegate namer.Namer +} + +// Name provides the requested name for a type. +func (n *ExceptionNamer) Name(t *types.Type) string { + key := n.KeyFunc(t) + if exception, ok := n.Exceptions[key]; ok { + return exception + } + return n.Delegate.Name(t) +} + +// DefaultNameSystem returns the default name system for ordering the types to be +// processed by the generators in this package. +func DefaultNameSystem() string { + return "public" +} +func targetForGroup(gv clientgentypes.GroupVersion, typeList []*types.Type, clientsetDir, clientsetPkg string, groupPkgName string, groupGoName string, apiPath string, inputPkg string, boilerplate []byte, protoPkg clientgentypes.ProtobufPackage) generator.Target { + + subdir := []string{"typed", strings.ToLower(groupPkgName), strings.ToLower(gv.Version.NonEmpty())} + gvDir := filepath.Join(clientsetDir, filepath.Join(subdir...)) + gvPkg := path.Join(clientsetPkg, path.Join(subdir...)) + + return &generator.SimpleTarget{ + PkgName: strings.ToLower(gv.Version.NonEmpty()), + PkgPath: gvPkg, + PkgDir: gvDir, + HeaderComment: boilerplate, + PkgDocComment: []byte("// This package has the automatically generated typed clients.\n"), + // GeneratorsFunc returns a list of generators. Each generator makes a + // single file. + GeneratorsFunc: func(c *generator.Context) (generators []generator.Generator) { + generators = []generator.Generator{ + // Always generate a "doc.go" file. + generator.GoGenerator{OutputFilename: "doc.go"}, + } + // Since we want a file per type that we generate a client for, we + // have to provide a function for this. + for _, t := range typeList { + generators = append(generators, &genClientForType{ + GoGenerator: generator.GoGenerator{ + OutputFilename: strings.ToLower(c.Namers["private"].Name(t)) + ".go", + }, + outputPackage: gvPkg, + inputPackage: inputPkg, + group: gv.Group.NonEmpty(), + version: gv.Version.String(), + groupGoName: groupGoName, + typeToMatch: t, + imports: generator.NewImportTrackerForPackage(gvPkg), + protoPackage: protoPkg, + }) + } + + generators = append(generators, &genGroup{ + GoGenerator: generator.GoGenerator{ + OutputFilename: groupPkgName + "_client.go", + }, + outputPackage: gvPkg, + inputPackage: inputPkg, + clientsetPackage: clientsetPkg, + group: gv.Group.NonEmpty(), + version: gv.Version.String(), + groupGoName: groupGoName, + apiPath: apiPath, + types: typeList, + imports: generator.NewImportTrackerForPackage(gvPkg), + }) + + expansionFileName := "generated_expansion.go" + generators = append(generators, &genExpansion{ + groupPackagePath: gvDir, + GoGenerator: generator.GoGenerator{ + OutputFilename: expansionFileName, + }, + types: typeList, + }) + + return generators + }, + FilterFunc: func(c *generator.Context, t *types.Type) bool { + return util.MustParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...)).GenerateClient + }, + } +} + +func targetForClientset(args *args.Args, clientsetDir, clientsetPkg string, groupGoNames map[clientgentypes.GroupVersion]string, boilerplate []byte) generator.Target { + return &generator.SimpleTarget{ + PkgName: args.ClientsetName, + PkgPath: clientsetPkg, + PkgDir: clientsetDir, + HeaderComment: boilerplate, + // GeneratorsFunc returns a list of generators. Each generator generates a + // single file. + GeneratorsFunc: func(c *generator.Context) (generators []generator.Generator) { + generators = []generator.Generator{ + &genClientset{ + GoGenerator: generator.GoGenerator{ + OutputFilename: "clientset.go", + }, + groups: args.Groups, + groupGoNames: groupGoNames, + clientsetPackage: clientsetPkg, + imports: generator.NewImportTrackerForPackage(clientsetPkg), + }, + } + return generators + }, + } +} + +func targetForScheme(args *args.Args, clientsetDir, clientsetPkg string, groupGoNames map[clientgentypes.GroupVersion]string, boilerplate []byte) generator.Target { + schemeDir := filepath.Join(clientsetDir, "scheme") + schemePkg := path.Join(clientsetPkg, "scheme") + + // create runtime.Registry for internal client because it has to know about group versions + internalClient := false +NextGroup: + for _, group := range args.Groups { + for _, v := range group.Versions { + if v.String() == "" { + internalClient = true + break NextGroup + } + } + } + + return &generator.SimpleTarget{ + PkgName: "scheme", + PkgPath: schemePkg, + PkgDir: schemeDir, + HeaderComment: boilerplate, + PkgDocComment: []byte("// This package contains the scheme of the automatically generated clientset.\n"), + // GeneratorsFunc returns a list of generators. Each generator generates a + // single file. + GeneratorsFunc: func(c *generator.Context) (generators []generator.Generator) { + generators = []generator.Generator{ + // Always generate a "doc.go" file. + generator.GoGenerator{OutputFilename: "doc.go"}, + + &scheme.GenScheme{ + GoGenerator: generator.GoGenerator{ + OutputFilename: "register.go", + }, + InputPackages: args.GroupVersionPackages(), + OutputPkg: schemePkg, + OutputPath: schemeDir, + Groups: args.Groups, + GroupGoNames: groupGoNames, + ImportTracker: generator.NewImportTrackerForPackage(schemePkg), + CreateRegistry: internalClient, + }, + } + return generators + }, + } +} + +// applyGroupOverrides applies group name overrides to each package, if applicable. If there is a +// comment of the form "// +groupName=somegroup" or "// +groupName=somegroup.foo.bar.io", use the +// first field (somegroup) as the name of the group in Go code, e.g. as the func name in a clientset. +// +// If the first field of the groupName is not unique within the clientset, use "// +groupName=unique +func applyGroupOverrides(universe types.Universe, args *args.Args) error { + // Create a map from "old GV" to "new GV" so we know what changes we need to make. + changes := make(map[clientgentypes.GroupVersion]clientgentypes.GroupVersion) + for gv, inputDir := range args.GroupVersionPackages() { + p := universe.Package(inputDir) + override, err := genutil.ExtractCommentTagsWithoutArguments("+", []string{"groupName"}, p.Comments) + if err != nil { + return fmt.Errorf("cannot extract groupName tags: %w", err) + } + if override["groupName"] != nil { + newGV := clientgentypes.GroupVersion{ + Group: clientgentypes.Group(override["groupName"][0]), + Version: gv.Version, + } + changes[gv] = newGV + } + } + + // Modify args.Groups based on the groupName overrides. + newGroups := make([]clientgentypes.GroupVersions, 0, len(args.Groups)) + for _, gvs := range args.Groups { + if len(gvs.Versions) == 0 { + return fmt.Errorf("group %q has no versions", gvs.Group.String()) + } + gv := clientgentypes.GroupVersion{ + Group: gvs.Group, + Version: gvs.Versions[0].Version, // we only need a version, and the first will do + } + if newGV, ok := changes[gv]; ok { + // There's an override, so use it. + newGVS := clientgentypes.GroupVersions{ + PackageName: gvs.PackageName, + Group: newGV.Group, + Versions: gvs.Versions, + } + newGroups = append(newGroups, newGVS) + } else { + // No override. + newGroups = append(newGroups, gvs) + } + } + args.Groups = newGroups + return nil +} + +// Because we try to assemble inputs from an input-base and a set of +// group-version arguments, sometimes that comes in as a filesystem path. This +// function rewrites them all as their canonical Go import-paths. +// +// TODO: Change this tool to just take inputs as Go "patterns" like every other +// gengo tool, then extract GVs from those. +func sanitizePackagePaths(context *generator.Context, args *args.Args) error { + for i := range args.Groups { + pkg := &args.Groups[i] + for j := range pkg.Versions { + ver := &pkg.Versions[j] + input := ver.Package + p := context.Universe[input] + if p == nil || p.Name == "" { + pkgs, err := context.FindPackages(input) + if err != nil { + return fmt.Errorf("can't find input package %q: %w", input, err) + } + p = context.Universe[pkgs[0]] + if p == nil { + return fmt.Errorf("can't find input package %q in universe", input) + } + ver.Package = p.Path + } + } + } + return nil +} + +// GetTargets makes the client target definition. +func GetTargets(context *generator.Context, args *args.Args) []generator.Target { + boilerplate, err := gengo.GoBoilerplate(args.GoHeaderFile, "", gengo.StdGeneratedBy) + if err != nil { + klog.Fatalf("Failed loading boilerplate: %v", err) + } + + includedTypesOverrides := args.IncludedTypesOverrides + + if err := sanitizePackagePaths(context, args); err != nil { + klog.Fatalf("cannot sanitize inputs: %v", err) + } + if err := applyGroupOverrides(context.Universe, args); err != nil { + klog.Fatalf("cannot apply group overrides: %v", err) + } + + gvToTypes := map[clientgentypes.GroupVersion][]*types.Type{} + groupGoNames := make(map[clientgentypes.GroupVersion]string) + for gv, inputDir := range args.GroupVersionPackages() { + p := context.Universe.Package(inputDir) + + // If there's a comment of the form "// +groupGoName=SomeUniqueShortName", use that as + // the Go group identifier in CamelCase. It defaults + groupGoNames[gv] = namer.IC(strings.Split(gv.Group.NonEmpty(), ".")[0]) + override, err := genutil.ExtractCommentTagsWithoutArguments("+", []string{"groupGoName"}, p.Comments) + if err != nil { + klog.Fatalf("cannot extract groupGoName tags: %v", err) + } + if override["groupGoName"] != nil { + groupGoNames[gv] = namer.IC(override["groupGoName"][0]) + } + + for n, t := range p.Types { + // filter out types which are not included in user specified overrides. + typesOverride, ok := includedTypesOverrides[gv] + if ok { + found := false + for _, typeStr := range typesOverride { + if typeStr == n { + found = true + break + } + } + if !found { + continue + } + } else { + // User has not specified any override for this group version. + // filter out types which don't have genclient. + if tags := util.MustParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...)); !tags.GenerateClient { + continue + } + } + if _, found := gvToTypes[gv]; !found { + gvToTypes[gv] = []*types.Type{} + } + gvToTypes[gv] = append(gvToTypes[gv], t) + } + } + + clientsetDir := filepath.Join(args.OutputDir, args.ClientsetName) + clientsetPkg := path.Join(args.OutputPkg, args.ClientsetName) + + var targetList []generator.Target + + targetList = append(targetList, + targetForClientset(args, clientsetDir, clientsetPkg, groupGoNames, boilerplate)) + targetList = append(targetList, + targetForScheme(args, clientsetDir, clientsetPkg, groupGoNames, boilerplate)) + if args.FakeClient { + targetList = append(targetList, + fake.TargetForClientset(args, clientsetDir, clientsetPkg, groupGoNames, boilerplate)) + } + + // If --clientset-only=true, we don't regenerate the individual typed clients. + if args.ClientsetOnly { + return []generator.Target(targetList) + } + + orderer := namer.Orderer{Namer: namer.NewPrivateNamer(0)} + gvPackages := args.GroupVersionPackages() + for _, group := range args.Groups { + for _, version := range group.Versions { + gv := clientgentypes.GroupVersion{Group: group.Group, Version: version.Version} + types := gvToTypes[gv] + inputPath := gvPackages[gv] + protoPackage := clientgentypes.NewProtobufPackage(args.ProtoBase, group.PackageName, version.Version.String()) + targetList = append(targetList, + targetForGroup( + gv, orderer.OrderTypes(types), clientsetDir, clientsetPkg, + group.PackageName, groupGoNames[gv], args.ClientsetAPIPath, + inputPath, boilerplate, protoPackage)) + if args.FakeClient { + targetList = append(targetList, + fake.TargetForGroup(gv, orderer.OrderTypes(types), clientsetDir, clientsetPkg, group.PackageName, groupGoNames[gv], inputPath, boilerplate)) + } + } + } + + return targetList +} diff --git a/code-generator/cmd/client-gen/generators/fake/fake_client_generator.go b/code-generator/cmd/client-gen/generators/fake/fake_client_generator.go new file mode 100644 index 000000000..b36d837ab --- /dev/null +++ b/code-generator/cmd/client-gen/generators/fake/fake_client_generator.go @@ -0,0 +1,137 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Portions Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved. + +Modified from the original to support gRPC transport. +Origin: https://github.com/kubernetes/code-generator/blob/v0.34.1/cmd/client-gen/generators/fake/fake_client_generator.go +*/ + +package fake + +import ( + "path" + "path/filepath" + "strings" + + "k8s.io/gengo/v2/generator" + "k8s.io/gengo/v2/types" + + "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/args" + scheme "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/generators/scheme" + "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/generators/util" + clientgentypes "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/types" +) + +func TargetForGroup(gv clientgentypes.GroupVersion, typeList []*types.Type, clientsetDir, clientsetPkg string, groupPkgName string, groupGoName string, inputPkg string, boilerplate []byte) generator.Target { + // TODO: should make this a function, called by here and in client-generator.go + subdir := []string{"typed", strings.ToLower(groupPkgName), strings.ToLower(gv.Version.NonEmpty())} + outputDir := filepath.Join(clientsetDir, filepath.Join(subdir...), "fake") + outputPkg := path.Join(clientsetPkg, path.Join(subdir...), "fake") + realClientPkg := path.Join(clientsetPkg, path.Join(subdir...)) + + return &generator.SimpleTarget{ + PkgName: "fake", + PkgPath: outputPkg, + PkgDir: outputDir, + HeaderComment: boilerplate, + PkgDocComment: []byte("// Package fake has the automatically generated clients.\n"), + // GeneratorsFunc returns a list of generators. Each generator makes a + // single file. + GeneratorsFunc: func(c *generator.Context) (generators []generator.Generator) { + generators = []generator.Generator{ + // Always generate a "doc.go" file. + generator.GoGenerator{OutputFilename: "doc.go"}, + } + // Since we want a file per type that we generate a client for, we + // have to provide a function for this. + for _, t := range typeList { + generators = append(generators, &genFakeForType{ + GoGenerator: generator.GoGenerator{ + OutputFilename: "fake_" + strings.ToLower(c.Namers["private"].Name(t)) + ".go", + }, + outputPackage: outputPkg, + realClientPackage: realClientPkg, + inputPackage: inputPkg, + version: gv.Version.String(), + groupGoName: groupGoName, + typeToMatch: t, + imports: generator.NewImportTrackerForPackage(outputPkg), + }) + } + + generators = append(generators, &genFakeForGroup{ + GoGenerator: generator.GoGenerator{ + OutputFilename: "fake_" + groupPkgName + "_client.go", + }, + outputPackage: outputPkg, + realClientPackage: realClientPkg, + version: gv.Version.String(), + groupGoName: groupGoName, + types: typeList, + imports: generator.NewImportTrackerForPackage(outputPkg), + }) + return generators + }, + FilterFunc: func(c *generator.Context, t *types.Type) bool { + return util.MustParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...)).GenerateClient + }, + } +} + +func TargetForClientset(args *args.Args, clientsetDir, clientsetPkg string, groupGoNames map[clientgentypes.GroupVersion]string, boilerplate []byte) generator.Target { + return &generator.SimpleTarget{ + // TODO: we'll generate fake clientset for different release in the future. + // Package name and path are hard coded for now. + PkgName: "fake", + PkgPath: path.Join(clientsetPkg, "fake"), + PkgDir: filepath.Join(clientsetDir, "fake"), + HeaderComment: boilerplate, + PkgDocComment: []byte("// This package has the automatically generated fake clientset.\n"), + // GeneratorsFunc returns a list of generators. Each generator generates a + // single file. + GeneratorsFunc: func(c *generator.Context) (generators []generator.Generator) { + generators = []generator.Generator{ + // Always generate a "doc.go" file. + generator.GoGenerator{OutputFilename: "doc.go"}, + + &genClientset{ + GoGenerator: generator.GoGenerator{ + OutputFilename: "clientset_generated.go", + }, + groups: args.Groups, + groupGoNames: groupGoNames, + fakeClientsetPackage: clientsetPkg, + imports: generator.NewImportTrackerForPackage(clientsetPkg), + realClientsetPackage: clientsetPkg, + }, + &scheme.GenScheme{ + GoGenerator: generator.GoGenerator{ + OutputFilename: "register.go", + }, + InputPackages: args.GroupVersionPackages(), + OutputPkg: clientsetPkg, + Groups: args.Groups, + GroupGoNames: groupGoNames, + ImportTracker: generator.NewImportTrackerForPackage(clientsetPkg), + PrivateScheme: true, + }, + } + return generators + }, + } +} diff --git a/code-generator/cmd/client-gen/generators/fake/generator_fake_for_clientset.go b/code-generator/cmd/client-gen/generators/fake/generator_fake_for_clientset.go new file mode 100644 index 000000000..ed2e007d0 --- /dev/null +++ b/code-generator/cmd/client-gen/generators/fake/generator_fake_for_clientset.go @@ -0,0 +1,203 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Portions Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved. + +Modified from the original to support gRPC transport. +Origin: https://github.com/kubernetes/code-generator/blob/v0.34.1/cmd/client-gen/generators/fake/generator_fake_for_clientset.go +*/ + +package fake + +import ( + "fmt" + "io" + "path" + "strings" + + clientgentypes "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/types" + "k8s.io/gengo/v2/generator" + "k8s.io/gengo/v2/namer" + "k8s.io/gengo/v2/types" +) + +// genClientset generates a package for a clientset. +type genClientset struct { + generator.GoGenerator + groups []clientgentypes.GroupVersions + groupGoNames map[clientgentypes.GroupVersion]string + fakeClientsetPackage string // must be a Go import-path + imports namer.ImportTracker + clientsetGenerated bool + // the import path of the generated real clientset. + realClientsetPackage string // must be a Go import-path +} + +var _ generator.Generator = &genClientset{} + +func (g *genClientset) Namers(c *generator.Context) namer.NameSystems { + return namer.NameSystems{ + "raw": namer.NewRawNamer(g.fakeClientsetPackage, g.imports), + } +} + +// We only want to call GenerateType() once. +func (g *genClientset) Filter(c *generator.Context, t *types.Type) bool { + ret := !g.clientsetGenerated + g.clientsetGenerated = true + return ret +} + +func (g *genClientset) Imports(c *generator.Context) (imports []string) { + imports = append(imports, g.imports.ImportLines()...) + for _, group := range g.groups { + for _, version := range group.Versions { + groupClientPackage := path.Join(g.fakeClientsetPackage, "typed", strings.ToLower(group.PackageName), strings.ToLower(version.NonEmpty())) + fakeGroupClientPackage := path.Join(groupClientPackage, "fake") + + groupAlias := strings.ToLower(g.groupGoNames[clientgentypes.GroupVersion{Group: group.Group, Version: version.Version}]) + imports = append(imports, fmt.Sprintf("%s%s \"%s\"", groupAlias, strings.ToLower(version.NonEmpty()), groupClientPackage)) + imports = append(imports, fmt.Sprintf("fake%s%s \"%s\"", groupAlias, strings.ToLower(version.NonEmpty()), fakeGroupClientPackage)) + } + } + // the package that has the clientset Interface + imports = append(imports, fmt.Sprintf("clientset \"%s\"", g.realClientsetPackage)) + // imports for the code in commonTemplate + imports = append(imports, + "k8s.io/client-go/testing", + "k8s.io/client-go/discovery", + "fakediscovery \"k8s.io/client-go/discovery/fake\"", + "k8s.io/apimachinery/pkg/runtime", + "k8s.io/apimachinery/pkg/watch", + "metav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"", + ) + + return +} + +func (g *genClientset) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error { + // TODO: We actually don't need any type information to generate the clientset, + // perhaps we can adapt the go2ild framework to this kind of usage. + sw := generator.NewSnippetWriter(w, c, "$", "$") + + allGroups := clientgentypes.ToGroupVersionInfo(g.groups, g.groupGoNames) + + mCommon := map[string]interface{}{ + "grpc": c.Universe.Type(types.Name{Package: "google.golang.org/grpc", Name: "ClientConnInterface"}), + } + + sw.Do(common, mCommon) + + sw.Do(checkImpl, nil) + + for _, group := range allGroups { + m := map[string]interface{}{ + "group": group.Group, + "version": group.Version, + "PackageAlias": group.PackageAlias, + "GroupGoName": group.GroupGoName, + "Version": namer.IC(group.Version.String()), + } + + sw.Do(clientsetInterfaceImplTemplate, m) + } + + return sw.Error() +} + +// This part of code is version-independent, unchanging. + +var common = ` +// NewSimpleClientset returns a clientset that will respond with the provided objects. +// It's backed by a very simple object tracker that processes creates, updates and deletions as-is, +// without applying any field management, validations and/or defaults. It shouldn't be considered a replacement +// for a real clientset and is mostly useful in simple unit tests. +func NewSimpleClientset(objects ...runtime.Object) *Clientset { + o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) + for _, obj := range objects { + if err := o.Add(obj); err != nil { + panic(err) + } + } + + cs := &Clientset{tracker: o} + cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} + cs.AddReactor("*", "*", testing.ObjectReaction(o)) + cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { + var opts metav1.ListOptions + if watchAction, ok := action.(testing.WatchActionImpl); ok { + opts = watchAction.ListOptions + } + gvr := action.GetResource() + ns := action.GetNamespace() + watch, err := o.Watch(gvr, ns, opts) + if err != nil { + return false, nil, err + } + return true, watch, nil + }) + + return cs +} + +// Clientset implements clientset.Interface. Meant to be embedded into a +// struct to get a default implementation. This makes faking out just the method +// you want to test easier. +type Clientset struct { + testing.Fake + discovery *fakediscovery.FakeDiscovery + tracker testing.ObjectTracker +} + +// ClientConn returns nil for fake clientsets. There is no real gRPC connection. +func (c *Clientset) ClientConn() $.grpc|raw$ { + return nil +} + +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + return c.discovery +} + +func (c *Clientset) Tracker() testing.ObjectTracker { + return c.tracker +} + +// IsWatchListSemanticsUnSupported informs the reflector that this client +// doesn't support WatchList semantics. +// +// This is a synthetic method whose sole purpose is to satisfy the optional +// interface check performed by the reflector. +// Returning true signals that WatchList can NOT be used. +// No additional logic is implemented here. +func (c *Clientset) IsWatchListSemanticsUnSupported() bool { + return true +} +` + +var checkImpl = ` +var ( + _ clientset.Interface = &Clientset{} + _ testing.FakeClient = &Clientset{} +) +` + +var clientsetInterfaceImplTemplate = ` +// $.GroupGoName$$.Version$ retrieves the $.GroupGoName$$.Version$Client +func (c *Clientset) $.GroupGoName$$.Version$() $.PackageAlias$.$.GroupGoName$$.Version$Interface { + return &fake$.PackageAlias$.Fake$.GroupGoName$$.Version${Fake: &c.Fake} +} +` diff --git a/code-generator/cmd/client-gen/generators/fake/generator_fake_for_group.go b/code-generator/cmd/client-gen/generators/fake/generator_fake_for_group.go new file mode 100644 index 000000000..68f507b7a --- /dev/null +++ b/code-generator/cmd/client-gen/generators/fake/generator_fake_for_group.go @@ -0,0 +1,130 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Portions Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved. + +Modified from the original to support gRPC transport. +Origin: https://github.com/kubernetes/code-generator/blob/v0.34.1/cmd/client-gen/generators/fake/generator_fake_for_group.go +*/ + +package fake + +import ( + "fmt" + "io" + "path" + "strings" + + "k8s.io/gengo/v2/generator" + "k8s.io/gengo/v2/namer" + "k8s.io/gengo/v2/types" + + "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/generators/util" +) + +// genFakeForGroup produces a file for a group client, e.g. ExtensionsClient for the extension group. +type genFakeForGroup struct { + generator.GoGenerator + outputPackage string // must be a Go import-path + realClientPackage string // must be a Go import-path + version string + groupGoName string + // types in this group + types []*types.Type + imports namer.ImportTracker + // If the genGroup has been called. This generator should only execute once. + called bool +} + +var _ generator.Generator = &genFakeForGroup{} + +// We only want to call GenerateType() once per group. +func (g *genFakeForGroup) Filter(c *generator.Context, t *types.Type) bool { + if !g.called { + g.called = true + return true + } + return false +} + +func (g *genFakeForGroup) Namers(c *generator.Context) namer.NameSystems { + return namer.NameSystems{ + "raw": namer.NewRawNamer(g.outputPackage, g.imports), + } +} + +func (g *genFakeForGroup) Imports(c *generator.Context) (imports []string) { + imports = g.imports.ImportLines() + if len(g.types) != 0 { + imports = append(imports, fmt.Sprintf("%s \"%s\"", strings.ToLower(path.Base(g.realClientPackage)), g.realClientPackage)) + } + return imports +} + +func (g *genFakeForGroup) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error { + sw := generator.NewSnippetWriter(w, c, "$", "$") + + m := map[string]interface{}{ + "GroupGoName": g.groupGoName, + "Version": namer.IC(g.version), + "Fake": c.Universe.Type(types.Name{Package: "k8s.io/client-go/testing", Name: "Fake"}), + "grpc": c.Universe.Type(types.Name{Package: "google.golang.org/grpc", Name: "ClientConnInterface"}), + } + + sw.Do(groupClientTemplate, m) + for _, t := range g.types { + tags, err := util.ParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...)) + if err != nil { + return fmt.Errorf("parse client-gen tags for %s: %w", t.Name.Name, err) + } + wrapper := map[string]interface{}{ + "type": t, + "GroupGoName": g.groupGoName, + "Version": namer.IC(g.version), + "realClientPackage": strings.ToLower(path.Base(g.realClientPackage)), + } + if tags.NonNamespaced { + sw.Do(getterImplNonNamespaced, wrapper) + continue + } + sw.Do(getterImplNamespaced, wrapper) + } + return sw.Error() +} + +var groupClientTemplate = ` +type Fake$.GroupGoName$$.Version$ struct { + *$.Fake|raw$ +} + +// ClientConn returns nil for fake clientsets. There is no real gRPC connection. +func (c *Fake$.GroupGoName$$.Version$) ClientConn() $.grpc|raw$ { + return nil +} +` + +var getterImplNamespaced = ` +func (c *Fake$.GroupGoName$$.Version$) $.type|publicPlural$(namespace string) $.realClientPackage$.$.type|public$Interface { + return newFake$.type|publicPlural$(c, namespace) +} +` + +var getterImplNonNamespaced = ` +func (c *Fake$.GroupGoName$$.Version$) $.type|publicPlural$() $.realClientPackage$.$.type|public$Interface { + return newFake$.type|publicPlural$(c) +} +` diff --git a/code-generator/cmd/client-gen/generators/fake/generator_fake_for_type.go b/code-generator/cmd/client-gen/generators/fake/generator_fake_for_type.go new file mode 100644 index 000000000..bcf34d1d1 --- /dev/null +++ b/code-generator/cmd/client-gen/generators/fake/generator_fake_for_type.go @@ -0,0 +1,347 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Portions Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved. + +Modified from the original to support gRPC transport. +Origin: https://github.com/kubernetes/code-generator/blob/v0.34.1/cmd/client-gen/generators/fake/generator_fake_for_type.go +*/ + +package fake + +import ( + "fmt" + "io" + + "golang.org/x/text/cases" + "golang.org/x/text/language" + + "k8s.io/gengo/v2/generator" + "k8s.io/gengo/v2/namer" + "k8s.io/gengo/v2/types" + + "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/generators/util" +) + +// genFakeForType produces a file for each top-level type. +type genFakeForType struct { + generator.GoGenerator + outputPackage string // Must be a Go import-path + realClientPackage string // Must be a Go import-path + version string + groupGoName string + inputPackage string + typeToMatch *types.Type + imports namer.ImportTracker +} + +var _ generator.Generator = &genFakeForType{} + +var titler = cases.Title(language.Und) + +// Filter ignores all but one type because we're making a single file per type. +func (g *genFakeForType) Filter(c *generator.Context, t *types.Type) bool { return t == g.typeToMatch } + +func (g *genFakeForType) Namers(c *generator.Context) namer.NameSystems { + return namer.NameSystems{ + "raw": namer.NewRawNamer(g.outputPackage, g.imports), + } +} + +func (g *genFakeForType) Imports(c *generator.Context) (imports []string) { + return g.imports.ImportLines() +} + +func genStatus(t *types.Type) bool { + // Default to true if we have a Status member + hasStatus := false + for _, m := range t.Members { + if m.Name == "Status" { + hasStatus = true + break + } + } + return hasStatus && !util.MustParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...)).NoStatus +} + +// GenerateType makes the body of a file implementing the individual typed client for type t. +func (g *genFakeForType) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error { + sw := generator.NewSnippetWriter(w, c, "$", "$") + tags, err := util.ParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...)) + if err != nil { + return fmt.Errorf("parse client-gen tags for %s: %w", t.Name.Name, err) + } + + const pkgClientGoTesting = "k8s.io/client-go/testing" + m := map[string]interface{}{ + "type": t, + "inputType": t, + "resultType": t, + "subresourcePath": "", + "namespaced": !tags.NonNamespaced, + "GroupGoName": g.groupGoName, + "Version": namer.IC(g.version), + "realClientInterface": c.Universe.Type(types.Name{Package: g.realClientPackage, Name: t.Name.Name + "Interface"}), + "SchemeGroupVersion": c.Universe.Type(types.Name{Package: t.Name.Package, Name: "SchemeGroupVersion"}), + "contextContext": c.Universe.Type(types.Name{Package: "context", Name: "Context"}), + "fmtErrorf": c.Universe.Type(types.Name{Package: "fmt", Name: "Errorf"}), + "GroupVersionResource": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/runtime/schema", Name: "GroupVersionResource"}), + "GroupVersionKind": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/runtime/schema", Name: "GroupVersionKind"}), + "watchInterface": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/watch", Name: "Interface"}), + "GetOptions": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/apis/meta/v1", Name: "GetOptions"}), + "ListOptions": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/apis/meta/v1", Name: "ListOptions"}), + "CreateOptions": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/apis/meta/v1", Name: "CreateOptions"}), + "DeleteOptions": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/apis/meta/v1", Name: "DeleteOptions"}), + "UpdateOptions": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/apis/meta/v1", Name: "UpdateOptions"}), + "PatchOptions": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/apis/meta/v1", Name: "PatchOptions"}), + "PatchType": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/types", Name: "PatchType"}), + "NewRootListActionWithOptions": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewRootListActionWithOptions"}), + "NewListActionWithOptions": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewListActionWithOptions"}), + "NewRootGetActionWithOptions": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewRootGetActionWithOptions"}), + "NewGetActionWithOptions": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewGetActionWithOptions"}), + "NewRootDeleteActionWithOptions": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewRootDeleteActionWithOptions"}), + "NewDeleteActionWithOptions": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewDeleteActionWithOptions"}), + "NewRootUpdateActionWithOptions": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewRootUpdateActionWithOptions"}), + "NewUpdateActionWithOptions": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewUpdateActionWithOptions"}), + "NewRootCreateActionWithOptions": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewRootCreateActionWithOptions"}), + "NewCreateActionWithOptions": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewCreateActionWithOptions"}), + "NewRootWatchActionWithOptions": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewRootWatchActionWithOptions"}), + "NewWatchActionWithOptions": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewWatchActionWithOptions"}), + "NewUpdateSubresourceActionWithOptions": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewUpdateSubresourceActionWithOptions"}), + "NewRootUpdateSubresourceActionWithOptions": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewRootUpdateSubresourceActionWithOptions"}), + "NewPatchSubresourceActionWithOptions": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewPatchSubresourceActionWithOptions"}), + "NewRootPatchSubresourceActionWithOptions": c.Universe.Function(types.Name{Package: pkgClientGoTesting, Name: "NewRootPatchSubresourceActionWithOptions"}), + } + + sw.Do(getterComment, m) + if tags.NonNamespaced { + sw.Do(getterNonNamespaced, m) + } else { + sw.Do(getterNamespaced, m) + } + + if tags.NonNamespaced { + sw.Do(structTemplateNonNamespaced, m) + sw.Do(constructorTemplateNonNamespaced, m) + } else { + sw.Do(structTemplateNamespaced, m) + sw.Do(constructorTemplateNamespaced, m) + } + sw.Do(structHelpers, m) + + if tags.HasVerb("get") { + sw.Do(getTemplate, m) + } + + if tags.HasVerb("list") { + sw.Do(listTemplate, m) + } + + if tags.HasVerb("watch") { + sw.Do(watchTemplate, m) + } + + if tags.HasVerb("create") { + sw.Do(createTemplate, m) + } + + if tags.HasVerb("update") { + sw.Do(updateTemplate, m) + } + + if genStatus(t) && tags.HasVerb("updateStatus") { + sw.Do(updateStatusTemplate, m) + } + + if tags.HasVerb("delete") { + sw.Do(deleteTemplate, m) + } + + if tags.HasVerb("patch") { + sw.Do(patchTemplate, m) + } + + return sw.Error() +} + +var getterComment = ` +// $.type|publicPlural$Getter has a method to return a $.type|public$Interface. +// A group's client should implement this interface.` + +var getterNamespaced = ` +type $.type|publicPlural$Getter interface { + $.type|publicPlural$(namespace string) $.realClientInterface|raw$ +} +` + +var getterNonNamespaced = ` +type $.type|publicPlural$Getter interface { + $.type|publicPlural$() $.realClientInterface|raw$ +} +` + +var structTemplateNamespaced = ` +// fake$.type|publicPlural$ implements $.type|public$Interface +type fake$.type|publicPlural$ struct { + Fake *Fake$.GroupGoName$$.Version$ + namespace string +} +` + +var structTemplateNonNamespaced = ` +// fake$.type|publicPlural$ implements $.type|public$Interface +type fake$.type|publicPlural$ struct { + Fake *Fake$.GroupGoName$$.Version$ +} +` + +var constructorTemplateNamespaced = ` +// newFake$.type|publicPlural$ returns a fake$.type|publicPlural$ +func newFake$.type|publicPlural$(fake *Fake$.GroupGoName$$.Version$, namespace string) $.realClientInterface|raw$ { + return &fake$.type|publicPlural${ + Fake: fake, + namespace: namespace, + } +} +` + +var constructorTemplateNonNamespaced = ` +// newFake$.type|publicPlural$ returns a fake$.type|publicPlural$ +func newFake$.type|publicPlural$(fake *Fake$.GroupGoName$$.Version$) $.realClientInterface|raw$ { + return &fake$.type|publicPlural${ + Fake: fake, + } +} +` + +var structHelpers = ` +func (c *fake$.type|publicPlural$) Resource() $.GroupVersionResource|raw$ { + return $.SchemeGroupVersion|raw$.WithResource("$.type|resource$") +} + +func (c *fake$.type|publicPlural$) Kind() $.GroupVersionKind|raw$ { + return $.SchemeGroupVersion|raw$.WithKind("$.type|singularKind$") +} + +func (c *fake$.type|publicPlural$) GetNamespace() string { + if c == nil { + return "" + } + return $if .namespaced$c.namespace$else$""$end$ +} +` + +var listTemplate = ` +// List takes label and field selectors, and returns the list of $.type|publicPlural$ that match those selectors. +func (c *fake$.type|publicPlural$) List(ctx $.contextContext|raw$, opts $.ListOptions|raw$) (result *$.type|raw$List, err error) { + emptyResult := &$.type|raw$List{} + obj, err := c.Fake. + $if .namespaced$Invokes($.NewListActionWithOptions|raw$(c.Resource(), c.Kind(), c.GetNamespace(), opts), emptyResult) + $else$Invokes($.NewRootListActionWithOptions|raw$(c.Resource(), c.Kind(), opts), emptyResult)$end$ + if obj == nil { + return emptyResult, err + } + return obj.(*$.type|raw$List), err +} +` + +var getTemplate = ` +// Get takes name of the $.type|public$, and returns the corresponding $.resultType|public$ object, and an error if there is any. +func (c *fake$.type|publicPlural$) Get(ctx $.contextContext|raw$, name string, opts $.GetOptions|raw$) (result *$.resultType|raw$, err error) { + emptyResult := &$.resultType|raw${} + obj, err := c.Fake. + $if .namespaced$Invokes($.NewGetActionWithOptions|raw$(c.Resource(), c.GetNamespace(), name, opts), emptyResult) + $else$Invokes($.NewRootGetActionWithOptions|raw$(c.Resource(), name, opts), emptyResult)$end$ + if obj == nil { + return emptyResult, err + } + return obj.(*$.resultType|raw$), err +} +` + +var deleteTemplate = ` +// Delete takes name of the $.type|private$ and deletes it. Returns an error if one occurs. +func (c *fake$.type|publicPlural$) Delete(ctx $.contextContext|raw$, name string, opts $.DeleteOptions|raw$) error { + _, err := c.Fake. + $if .namespaced$Invokes($.NewDeleteActionWithOptions|raw$(c.Resource(), c.GetNamespace(), name, opts), &$.type|raw${}) + $else$Invokes($.NewRootDeleteActionWithOptions|raw$(c.Resource(), name, opts), &$.type|raw${})$end$ + return err +} +` + +var createTemplate = ` +// Create takes the representation of a $.inputType|private$ and creates it. Returns the server's representation of the $.resultType|private$, and an error, if there is any. +func (c *fake$.type|publicPlural$) Create(ctx $.contextContext|raw$, $.inputType|private$ *$.inputType|raw$, opts $.CreateOptions|raw$) (result *$.resultType|raw$, err error) { + emptyResult := &$.resultType|raw${} + obj, err := c.Fake. + $if .namespaced$Invokes($.NewCreateActionWithOptions|raw$(c.Resource(), c.GetNamespace(), $.inputType|private$, opts), emptyResult) + $else$Invokes($.NewRootCreateActionWithOptions|raw$(c.Resource(), $.inputType|private$, opts), emptyResult)$end$ + if obj == nil { + return emptyResult, err + } + return obj.(*$.resultType|raw$), err +} +` + +var updateTemplate = ` +// Update takes the representation of a $.inputType|private$ and updates it. Returns the server's representation of the $.resultType|private$, and an error, if there is any. +func (c *fake$.type|publicPlural$) Update(ctx $.contextContext|raw$, $.inputType|private$ *$.inputType|raw$, opts $.UpdateOptions|raw$) (result *$.resultType|raw$, err error) { + emptyResult := &$.resultType|raw${} + obj, err := c.Fake. + $if .namespaced$Invokes($.NewUpdateActionWithOptions|raw$(c.Resource(), c.GetNamespace(), $.inputType|private$, opts), emptyResult) + $else$Invokes($.NewRootUpdateActionWithOptions|raw$(c.Resource(), $.inputType|private$, opts), emptyResult)$end$ + if obj == nil { + return emptyResult, err + } + return obj.(*$.resultType|raw$), err +} +` + +var updateStatusTemplate = ` +func (c *fake$.type|publicPlural$) UpdateStatus(ctx $.contextContext|raw$, $.type|private$ *$.type|raw$, opts $.UpdateOptions|raw$) (*$.type|raw$, error) { + obj, err := c.Fake. + $if .namespaced$Invokes($.NewUpdateSubresourceActionWithOptions|raw$(c.Resource(), "status", c.GetNamespace(), $.type|private$, opts), &$.type|raw${}) + $else$Invokes($.NewRootUpdateSubresourceActionWithOptions|raw$(c.Resource(), "status", $.type|private$, opts), &$.type|raw${})$end$ + if obj == nil { + return nil, err + } + return obj.(*$.type|raw$), err +} +` + +var watchTemplate = ` +// Watch returns a $.watchInterface|raw$ that watches the requested $.type|publicPlural$. +func (c *fake$.type|publicPlural$) Watch(ctx $.contextContext|raw$, opts $.ListOptions|raw$) ($.watchInterface|raw$, error) { + return c.Fake. + $if .namespaced$InvokesWatch($.NewWatchActionWithOptions|raw$($.SchemeGroupVersion|raw$.WithResource("$.type|resource$"), c.GetNamespace(), opts)) + $else$InvokesWatch($.NewRootWatchActionWithOptions|raw$($.SchemeGroupVersion|raw$.WithResource("$.type|resource$"), opts))$end$ +} +` + +var patchTemplate = ` +// Patch applies the patch and returns the patched $.resultType|private$. +func (c *fake$.type|publicPlural$) Patch(ctx $.contextContext|raw$, name string, pt $.PatchType|raw$, data []byte, opts $.PatchOptions|raw$, subresources ...string) (result *$.resultType|raw$, err error) { + emptyResult := &$.resultType|raw${} + obj, err := c.Fake. + $if .namespaced$Invokes($.NewPatchSubresourceActionWithOptions|raw$($.SchemeGroupVersion|raw$.WithResource("$.type|resource$"), c.GetNamespace(), name, pt, data, opts, subresources... ), emptyResult) + $else$Invokes($.NewRootPatchSubresourceActionWithOptions|raw$($.SchemeGroupVersion|raw$.WithResource("$.type|resource$"), name, pt, data, opts, subresources...), emptyResult)$end$ + if obj == nil { + return emptyResult, err + } + return obj.(*$.resultType|raw$), err +} +` diff --git a/code-generator/cmd/client-gen/generators/generator_for_clientset.go b/code-generator/cmd/client-gen/generators/generator_for_clientset.go new file mode 100644 index 000000000..b813e8f1b --- /dev/null +++ b/code-generator/cmd/client-gen/generators/generator_for_clientset.go @@ -0,0 +1,197 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Portions Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved. + +Modified from the original to support gRPC transport. +Origin: https://github.com/kubernetes/code-generator/blob/v0.34.1/cmd/client-gen/generators/generator_for_clientset.go +*/ + +// Package generators has the generators for the client-gen utility. +package generators + +import ( + "fmt" + "io" + "path" + "strings" + + clientgentypes "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/types" + + "k8s.io/gengo/v2/generator" + "k8s.io/gengo/v2/namer" + "k8s.io/gengo/v2/types" +) + +// genClientset generates a package for a clientset. +type genClientset struct { + generator.GoGenerator + groups []clientgentypes.GroupVersions + groupGoNames map[clientgentypes.GroupVersion]string + clientsetPackage string // must be a Go import-path + imports namer.ImportTracker + clientsetGenerated bool +} + +var _ generator.Generator = &genClientset{} + +func (g *genClientset) Namers(c *generator.Context) namer.NameSystems { + return namer.NameSystems{ + "raw": namer.NewRawNamer(g.clientsetPackage, g.imports), + } +} + +// We only want to call GenerateType() once. +func (g *genClientset) Filter(c *generator.Context, t *types.Type) bool { + ret := !g.clientsetGenerated + g.clientsetGenerated = true + return ret +} + +func (g *genClientset) Imports(c *generator.Context) (imports []string) { + imports = append(imports, g.imports.ImportLines()...) + for _, group := range g.groups { + for _, version := range group.Versions { + typedClientPath := path.Join(g.clientsetPackage, "typed", strings.ToLower(group.PackageName), strings.ToLower(version.NonEmpty())) + groupAlias := strings.ToLower(g.groupGoNames[clientgentypes.GroupVersion{Group: group.Group, Version: version.Version}]) + imports = append(imports, fmt.Sprintf("%s%s \"%s\"", groupAlias, strings.ToLower(version.NonEmpty()), typedClientPath)) + } + } + return +} + +func (g *genClientset) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error { + // TODO: We actually don't need any type information to generate the clientset, + // perhaps we can adapt the go2ild framework to this kind of usage. + sw := generator.NewSnippetWriter(w, c, "$", "$") + + allGroups := clientgentypes.ToGroupVersionInfo(g.groups, g.groupGoNames) + m := map[string]interface{}{ + "allGroups": allGroups, + "fmtErrorf": c.Universe.Function(types.Name{Package: "fmt", Name: "Errorf"}), + "Config": c.Universe.Type(types.Name{Package: "github.com/nvidia/nvsentinel/client-go/nvgrpc", Name: "Config"}), + "ClientConnFor": c.Universe.Function(types.Name{Package: "github.com/nvidia/nvsentinel/client-go/nvgrpc", Name: "ClientConnFor"}), + "ClientConnInterface": c.Universe.Type(types.Name{Package: "google.golang.org/grpc", Name: "ClientConnInterface"}), + } + + sw.Do(clientsetInterface, m) + sw.Do(clientsetTemplate, m) + for _, g := range allGroups { + sw.Do(clientsetInterfaceImplTemplate, g) + } + sw.Do(newClientsetForConfigTemplate, m) + sw.Do(newClientsetForConfigAndClientTemplate, m) + sw.Do(newClientsetForConfigOrDieTemplate, m) + sw.Do(newClientsetForGrpcClientTemplate, m) + + return sw.Error() +} + +var clientsetInterface = ` +type Interface interface { + $range .allGroups$$.GroupGoName$$.Version$() $.PackageAlias$.$.GroupGoName$$.Version$Interface + $end$ +} +` +var clientsetTemplate = ` +// Clientset contains the clients for groups. +type Clientset struct { + $range .allGroups$$.LowerCaseGroupGoName$$.Version$ *$.PackageAlias$.$.GroupGoName$$.Version$Client + $end$ +} +` + +var clientsetInterfaceImplTemplate = ` +// $.GroupGoName$$.Version$ retrieves the $.GroupGoName$$.Version$Client +func (c *Clientset) $.GroupGoName$$.Version$() $.PackageAlias$.$.GroupGoName$$.Version$Interface { + return c.$.LowerCaseGroupGoName$$.Version$ +} +` + +var newClientsetForConfigTemplate = ` +// NewForConfig creates a new Clientset for the given config. +// NewForConfig is equivalent to NewForConfigAndClient(c, clientConn), +// where clientConn was generated with nvgrpc.ClientConnFor(c). +// +// If you need to customize the connection (e.g. set a logger), +// use nvgrpc.ClientConnFor() manually and pass the connection to NewForConfigAndClient. +func NewForConfig(c *$.Config|raw$) (*Clientset, error) { + if c == nil { + return nil, $.fmtErrorf|raw$("config cannot be nil") + } + + configShallowCopy := *c // Shallow copy to avoid mutation + conn, err := $.ClientConnFor|raw$(&configShallowCopy) + if err != nil { + return nil, err + } + + cs, err := NewForConfigAndClient(&configShallowCopy, conn) + if err != nil { + // Close connection to prevent resource leak + _ = conn.Close() + return nil, err + } + return cs, nil +} +` + +var newClientsetForConfigAndClientTemplate = ` +// NewForConfigAndClient creates a new Clientset for the given config and gRPC client connection. +// The provided gRPC client connection provided takes precedence over the configured transport values. +func NewForConfigAndClient(c *$.Config|raw$, conn $.ClientConnInterface|raw$) (*Clientset, error) { + if c == nil { + return nil, $.fmtErrorf|raw$("config cannot be nil") + } + if conn == nil { + return nil, $.fmtErrorf|raw$("gRPC connection cannot be nil") + } + + configShallowCopy := *c // Shallow copy to avoid mutation + + var cs Clientset + var err error +$range .allGroups$ cs.$.LowerCaseGroupGoName$$.Version$, err = $.PackageAlias$.NewForConfigAndClient(&configShallowCopy, conn) + if err != nil { + return nil, err + } +$end$ + return &cs, nil +} +` + +var newClientsetForConfigOrDieTemplate = ` +// NewForConfigOrDie creates a new Clientset for the given config and +// panics if there is an error in the config or connection setup. +func NewForConfigOrDie(c *$.Config|raw$) *Clientset { + cs, err := NewForConfig(c) + if err != nil { + panic(err) + } + return cs +} +` + +var newClientsetForGrpcClientTemplate = ` +// New creates a new Clientset for the given gRPC client connection. +func New(conn $.ClientConnInterface|raw$) *Clientset { + var cs Clientset +$range .allGroups$ cs.$.LowerCaseGroupGoName$$.Version$ = $.PackageAlias$.New(conn) +$end$ + return &cs +} +` diff --git a/code-generator/cmd/client-gen/generators/generator_for_expansion.go b/code-generator/cmd/client-gen/generators/generator_for_expansion.go new file mode 100644 index 000000000..91d4463f0 --- /dev/null +++ b/code-generator/cmd/client-gen/generators/generator_for_expansion.go @@ -0,0 +1,61 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Copied unmodified from the original. +Reason: The upstream package is internal and cannot be imported. +Origin: https://github.com/kubernetes/code-generator/blob/v0.34.1/cmd/client-gen/generators/generator_for_expansion.go +*/ + +// Package generators has the generators for the client-gen utility. +package generators + +import ( + "io" + "os" + "path/filepath" + "strings" + + "k8s.io/gengo/v2/generator" + "k8s.io/gengo/v2/types" +) + +// genExpansion produces a file for a group client, e.g. ExtensionsClient for the extension group. +type genExpansion struct { + generator.GoGenerator + groupPackagePath string + // types in a group + types []*types.Type +} + +// We only want to call GenerateType() once per group. +func (g *genExpansion) Filter(c *generator.Context, t *types.Type) bool { + return len(g.types) == 0 || t == g.types[0] +} + +func (g *genExpansion) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error { + sw := generator.NewSnippetWriter(w, c, "$", "$") + for _, t := range g.types { + if _, err := os.Stat(filepath.Join(g.groupPackagePath, strings.ToLower(t.Name.Name+"_expansion.go"))); os.IsNotExist(err) { + sw.Do(expansionInterfaceTemplate, t) + } + } + return sw.Error() +} + +var expansionInterfaceTemplate = ` +type $.|public$Expansion interface {} +` diff --git a/code-generator/cmd/client-gen/generators/generator_for_group.go b/code-generator/cmd/client-gen/generators/generator_for_group.go new file mode 100644 index 000000000..550673f01 --- /dev/null +++ b/code-generator/cmd/client-gen/generators/generator_for_group.go @@ -0,0 +1,228 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Portions Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved. + +Modified from the original to support gRPC transport. +Origin: https://github.com/kubernetes/code-generator/blob/v0.34.1/cmd/client-gen/generators/generator_for_group.go +*/ + +// Package generators has the generators for the client-gen utility. +package generators + +import ( + "fmt" + "io" + + genutil "k8s.io/code-generator/pkg/util" + "k8s.io/gengo/v2/generator" + "k8s.io/gengo/v2/namer" + "k8s.io/gengo/v2/types" + + "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/generators/util" +) + +// genGroup produces a file for a group client, e.g. ExtensionsClient for the extension group. +type genGroup struct { + generator.GoGenerator + outputPackage string + group string + version string + groupGoName string + apiPath string + // types in this group + types []*types.Type + imports namer.ImportTracker + inputPackage string + clientsetPackage string // must be a Go import-path + // If the genGroup has been called. This generator should only execute once. + called bool +} + +var _ generator.Generator = &genGroup{} + +// We only want to call GenerateType() once per group. +func (g *genGroup) Filter(c *generator.Context, t *types.Type) bool { + if !g.called { + g.called = true + return true + } + return false +} + +func (g *genGroup) Namers(c *generator.Context) namer.NameSystems { + return namer.NameSystems{ + "raw": namer.NewRawNamer(g.outputPackage, g.imports), + } +} + +func (g *genGroup) Imports(c *generator.Context) (imports []string) { + imports = append(imports, g.imports.ImportLines()...) + return +} + +func (g *genGroup) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error { + sw := generator.NewSnippetWriter(w, c, "$", "$") + + // allow user to define a group name that's different from the one parsed from the directory. + p := c.Universe.Package(g.inputPackage) + groupName := g.group + override, err := genutil.ExtractCommentTagsWithoutArguments("+", []string{"groupName"}, p.Comments) + if err != nil { + return err + } + if values, ok := override["groupName"]; ok { + groupName = values[0] + } + + m := map[string]interface{}{ + "version": g.version, + "groupName": groupName, + "GroupGoName": g.groupGoName, + "Version": namer.IC(g.version), + "types": g.types, + "ClientConnInterface": c.Universe.Type(types.Name{Package: "google.golang.org/grpc", Name: "ClientConnInterface"}), + "Config": c.Universe.Type(types.Name{Package: "github.com/nvidia/nvsentinel/client-go/nvgrpc", Name: "Config"}), + "ClientConnFor": c.Universe.Function(types.Name{Package: "github.com/nvidia/nvsentinel/client-go/nvgrpc", Name: "ClientConnFor"}), + "Logger": c.Universe.Type(types.Name{Package: "github.com/go-logr/logr", Name: "Logger"}), + "Discard": c.Universe.Function(types.Name{Package: "github.com/go-logr/logr", Name: "Discard"}), + "fmtErrorf": c.Universe.Function(types.Name{Package: "fmt", Name: "Errorf"}), + } + sw.Do(groupInterfaceTemplate, m) + sw.Do(groupClientTemplate, m) + for _, t := range g.types { + tags, err := util.ParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...)) + if err != nil { + return fmt.Errorf("parse client-gen tags for %s: %w", t.Name.Name, err) + } + wrapper := map[string]interface{}{ + "type": t, + "GroupGoName": g.groupGoName, + "Version": namer.IC(g.version), + } + if tags.NonNamespaced { + sw.Do(getterImplNonNamespaced, wrapper) + } else { + sw.Do(getterImplNamespaced, wrapper) + } + } + sw.Do(newClientForConfigTemplate, m) + sw.Do(newClientForConfigAndClientTemplate, m) + sw.Do(newClientForConfigOrDieTemplate, m) + sw.Do(newClientForGrpcConnTemplate, m) + sw.Do(getClientConn, m) + + return sw.Error() +} + +var groupInterfaceTemplate = ` +type $.GroupGoName$$.Version$Interface interface { + ClientConn() $.ClientConnInterface|raw$ + $range .types$ $.|publicPlural$Getter + $end$ +} +` + +var groupClientTemplate = ` +// $.GroupGoName$$.Version$Client is used to interact with features provided by the $.groupName$ group. +type $.GroupGoName$$.Version$Client struct { + conn $.ClientConnInterface|raw$ + logger $.Logger|raw$ +} +` + +var getterImplNamespaced = ` +func (c *$.GroupGoName$$.Version$Client) $.type|publicPlural$(namespace string) $.type|public$Interface { + return new$.type|publicPlural$(c, namespace) +} +` + +var getterImplNonNamespaced = ` +func (c *$.GroupGoName$$.Version$Client) $.type|publicPlural$() $.type|public$Interface { + return new$.type|publicPlural$(c) +} +` + +var newClientForConfigTemplate = ` +// NewForConfig creates a new $.GroupGoName$$.Version$Client for the given config. +// NewForConfig is equivalent to NewForConfigAndClient(c, clientConn), +// where clientConn was generated with nvgrpc.ClientConnFor(c). +func NewForConfig(c *$.Config|raw$) (*$.GroupGoName$$.Version$Client, error) { + if c == nil { + return nil, $.fmtErrorf|raw$("config cannot be nil") + } + + config := *c // Shallow copy to avoid mutation + conn, err := $.ClientConnFor|raw$(&config) + if err != nil { + return nil, err + } + + return NewForConfigAndClient(&config, conn) +} +` + +var newClientForConfigAndClientTemplate = ` +// NewForConfigAndClient creates a new $.GroupGoName$$.Version$Client for the given config and gRPC client connection. +// Note the grpc client connection provided takes precedence over the configured transport values. +func NewForConfigAndClient(c *$.Config|raw$, conn $.ClientConnInterface|raw$) (*$.GroupGoName$$.Version$Client, error) { + if c == nil { + return nil, $.fmtErrorf|raw$("config cannot be nil") + } + if conn == nil { + return nil, $.fmtErrorf|raw$("gRPC connection cannot be nil") + } + + return &$.GroupGoName$$.Version$Client{ + conn: conn, + logger: c.GetLogger().WithName("$.groupName$.$.version$"), + }, nil +} +` + +var newClientForConfigOrDieTemplate = ` +// NewForConfigOrDie creates a new $.GroupGoName$$.Version$Client for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *$.Config|raw$) *$.GroupGoName$$.Version$Client { + client, err := NewForConfig(c) + if err != nil { + panic(err) + } + return client +} +` + +var getClientConn = ` +// ClientConn returns a gRPC client connection that is used to communicate +// with API server by this client implementation. +func (c *$.GroupGoName$$.Version$Client) ClientConn() $.ClientConnInterface|raw$ { + if c == nil { + return nil + } + return c.conn +} +` + +var newClientForGrpcConnTemplate = ` +// New creates a new $.GroupGoName$$.Version$Client for the given gRPC client connection. +func New(c $.ClientConnInterface|raw$) *$.GroupGoName$$.Version$Client { + return &$.GroupGoName$$.Version$Client{ + conn: c, + logger: $.Discard|raw$(), + } +} +` diff --git a/code-generator/cmd/client-gen/generators/generator_for_type.go b/code-generator/cmd/client-gen/generators/generator_for_type.go new file mode 100644 index 000000000..4418c4408 --- /dev/null +++ b/code-generator/cmd/client-gen/generators/generator_for_type.go @@ -0,0 +1,418 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Portions Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved. + +Modified from the original to support gRPC transport. +Origin: https://github.com/kubernetes/code-generator/blob/v0.34.1/cmd/client-gen/generators/generator_for_type.go +*/ + +// Package generators has the generators for the client-gen utility. +package generators + +import ( + "fmt" + "io" + "path" + "strings" + + "golang.org/x/text/cases" + "golang.org/x/text/language" + + "k8s.io/gengo/v2/generator" + "k8s.io/gengo/v2/namer" + "k8s.io/gengo/v2/types" + + "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/generators/util" + clientgentypes "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/types" +) + +// genClientForType produces a file for each top-level type. +type genClientForType struct { + generator.GoGenerator + outputPackage string // must be a Go import-path + inputPackage string + group string + version string + groupGoName string + typeToMatch *types.Type + imports namer.ImportTracker + protoPackage clientgentypes.ProtobufPackage +} + +var _ generator.Generator = &genClientForType{} + +var titler = cases.Title(language.Und) + +// Filter ignores all but one type because we're making a single file per type. +func (g *genClientForType) Filter(c *generator.Context, t *types.Type) bool { + return t == g.typeToMatch +} + +func (g *genClientForType) Namers(c *generator.Context) namer.NameSystems { + return namer.NameSystems{ + "raw": namer.NewRawNamer(g.outputPackage, g.imports), + } +} + +func (g *genClientForType) Imports(c *generator.Context) (imports []string) { + return append( + g.imports.ImportLines(), + g.protoPackage.ImportLines()..., + ) +} + +// Ideally, we'd like genStatus to return true if there is a subresource path +// registered for "status" in the API server, but we do not have that +// information, so genStatus returns true if the type has a status field. +func genStatus(t *types.Type) bool { + // Default to true if we have a Status member + hasStatus := false + for _, m := range t.Members { + if m.Name == "Status" { + hasStatus = true + break + } + } + return hasStatus && !util.MustParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...)).NoStatus +} + +func (g *genClientForType) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error { + sw := generator.NewSnippetWriter(w, c, "$", "$") + pkg := path.Base(t.Name.Package) + tags, err := util.ParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...)) + if err != nil { + return fmt.Errorf("parse client-gen tags for %s: %w", t.Name.Name, err) + } + protoType := titler.String(t.Name.Name) + m := map[string]interface{}{ + "type": t, + "inputType": t, + "resultType": t, + "package": pkg, + "Package": namer.IC(pkg), + "namespaced": !tags.NonNamespaced, + "Group": namer.IC(g.group), + "GroupGoName": g.groupGoName, + "Version": namer.IC(g.version), + "ProtoType": protoType, + "context": c.Universe.Type(types.Name{Package: "context", Name: "Context"}), + "fmtErrorf": c.Universe.Function(types.Name{Package: "fmt", Name: "Errorf"}), + "metav1": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/apis/meta/v1", Name: "Option"}), + "GetOptions": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/apis/meta/v1", Name: "GetOptions"}), + "ListOptions": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/apis/meta/v1", Name: "ListOptions"}), + "CreateOptions": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/apis/meta/v1", Name: "CreateOptions"}), + "DeleteOptions": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/apis/meta/v1", Name: "DeleteOptions"}), + "UpdateOptions": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/apis/meta/v1", Name: "UpdateOptions"}), + "PatchOptions": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/apis/meta/v1", Name: "PatchOptions"}), + "PatchType": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/types", Name: "PatchType"}), + "runtime": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/runtime", Name: "Object"}), + "watchInterface": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/watch", Name: "Interface"}), + "logr": c.Universe.Type(types.Name{Package: "github.com/go-logr/logr", Name: "Logger"}), + "nvgrpc": c.Universe.Type(types.Name{Package: "github.com/nvidia/nvsentinel/client-go/nvgrpc", Name: "NewWatcher"}), + "pb": g.protoPackage.Alias, + "apiPackage": c.Universe.Type(types.Name{Package: g.inputPackage, Name: "Ignored"}), + "FromProto": c.Universe.Function(types.Name{Package: g.inputPackage, Name: "FromProto"}), + "FromProtoList": c.Universe.Function(types.Name{Package: g.inputPackage, Name: "FromProtoList"}), + "NewServiceClient": g.protoPackage.ServiceClientConstructorFor(protoType), + } + + sw.Do(getterComment, m) + if tags.NonNamespaced { + sw.Do(getterNonNamespaced, m) + } else { + sw.Do(getterNamespaced, m) + } + + sw.Do(generateInterfaceTemplate(tags, t), m) + + if tags.NonNamespaced { + sw.Do(structTemplateNonNamespaced, m) + sw.Do(constructorTemplateNonNamespaced, m) + } else { + sw.Do(structTemplateNamespaced, m) + sw.Do(constructorTemplateNamespaced, m) + } + + sw.Do(structHelpers, m) + + if tags.HasVerb("get") { + sw.Do(getTemplate, m) + } + + if tags.HasVerb("list") { + sw.Do(listTemplate, m) + } + + if tags.HasVerb("watch") { + sw.Do(watchTemplate, m) + sw.Do(watchAdapterTemplate, m) + } + + if tags.HasVerb("create") { + sw.Do(createTemplate, m) + } + + if tags.HasVerb("update") { + sw.Do(updateTemplate, m) + } + + if genStatus(t) && tags.HasVerb("updateStatus") { + sw.Do(updateStatusTemplate, m) + } + + if tags.HasVerb("delete") { + sw.Do(deleteTemplate, m) + } + + if tags.HasVerb("patch") { + sw.Do(patchTemplate, m) + } + + return sw.Error() +} + +func generateInterfaceTemplate(tags util.Tags, t *types.Type) string { + lines := []string{} + + lines = append(lines, ` +// $.type|public$Interface has methods to work with $.type|public$ resources. +type $.type|public$Interface interface {`) + + if tags.HasVerb("create") { + lines = append(lines, ` Create(ctx $.context|raw$, $.type|private$ *$.type|raw$, opts $.CreateOptions|raw$) (*$.type|raw$, error)`) + } + if tags.HasVerb("update") { + lines = append(lines, ` Update(ctx $.context|raw$, $.type|private$ *$.type|raw$, opts $.UpdateOptions|raw$) (*$.type|raw$, error)`) + } + if genStatus(t) && tags.HasVerb("updateStatus") { + lines = append(lines, ` UpdateStatus(ctx $.context|raw$, $.type|private$ *$.type|raw$, opts $.UpdateOptions|raw$) (*$.type|raw$, error)`) + } + if tags.HasVerb("delete") { + lines = append(lines, ` Delete(ctx $.context|raw$, name string, opts $.DeleteOptions|raw$) error`) + } + if tags.HasVerb("get") { + lines = append(lines, ` Get(ctx $.context|raw$, name string, opts $.GetOptions|raw$) (*$.type|raw$, error)`) + } + if tags.HasVerb("list") { + lines = append(lines, ` List(ctx $.context|raw$, opts $.ListOptions|raw$) (*$.type|raw$List, error)`) + } + if tags.HasVerb("watch") { + lines = append(lines, ` Watch(ctx $.context|raw$, opts $.ListOptions|raw$) ($.watchInterface|raw$, error)`) + } + if tags.HasVerb("patch") { + lines = append(lines, ` Patch(ctx $.context|raw$, name string, pt $.PatchType|raw$, data []byte, opts $.PatchOptions|raw$, subresources ...string) (result *$.type|raw$, err error)`) + } + + lines = append(lines, ` $.type|public$Expansion +} +`) + + return strings.Join(lines, "\n") +} + +var getterComment = ` +// $.type|publicPlural$Getter has a method to return a $.type|public$Interface. +// A group's client should implement this interface.` + +var getterNamespaced = ` +type $.type|publicPlural$Getter interface { + $.type|publicPlural$(namespace string) $.type|public$Interface +} +` + +var getterNonNamespaced = ` +type $.type|publicPlural$Getter interface { + $.type|publicPlural$() $.type|public$Interface +} +` + +var structTemplateNamespaced = ` +// $.type|allLowercasePlural$ implements $.type|public$Interface +type $.type|allLowercasePlural$ struct { + client $.pb$.$.ProtoType$ServiceClient + logger $.logr|raw$ + namespace string +} +` + +var structTemplateNonNamespaced = ` +// $.type|allLowercasePlural$ implements $.type|public$Interface +type $.type|allLowercasePlural$ struct { + client $.pb$.$.ProtoType$ServiceClient + logger $.logr|raw$ +} +` + +var constructorTemplateNamespaced = ` +// new$.type|publicPlural$ returns a $.type|allLowercasePlural$ +func new$.type|publicPlural$(c *$.GroupGoName$$.Version$Client, namespace string) *$.type|allLowercasePlural$ { + return &$.type|allLowercasePlural${ + client: $.NewServiceClient$(c.ClientConn()), + logger: c.logger.WithName("$.type|allLowercasePlural$"), + namespace: namespace, + } +} +` + +var constructorTemplateNonNamespaced = ` +// new$.type|publicPlural$ returns a $.type|allLowercasePlural$ +func new$.type|publicPlural$(c *$.GroupGoName$$.Version$Client) *$.type|allLowercasePlural$ { + return &$.type|allLowercasePlural${ + client: $.NewServiceClient$(c.ClientConn()), + logger: c.logger.WithName("$.type|allLowercasePlural$"), + } +} +` + +var structHelpers = ` +func (c *$.type|allLowercasePlural$) getNamespace() string { + if c == nil { + return "" + } + return $if .namespaced$c.namespace$else$""$end$ +} +` + +var listTemplate = ` +func (c *$.type|allLowercasePlural$) List(ctx $.context|raw$, opts $.ListOptions|raw$) (*$.type|raw$List, error) { + resp, err := c.client.List$.ProtoType$s(ctx, &$.pb$.List$.ProtoType$sRequest{ + Opts: &$.pb$.ListOptions{ + ResourceVersion: opts.ResourceVersion, + Namespace: c.getNamespace(), + }, + }) + if err != nil { + return nil, err + } + + list := $.FromProtoList|raw$(resp.Get$.ProtoType$List()) + if list == nil { + return nil, $.fmtErrorf|raw$("received nil $.type|public$ list from server for namespace %s", c.getNamespace()) + } + c.logger.V(5).Info("Listed $.type|public$s", + "namespace", c.getNamespace(), + "count", len(list.Items), + "resource-version", list.GetResourceVersion(), + ) + + return list, nil +} +` + +var getTemplate = ` +func (c *$.type|allLowercasePlural$) Get(ctx $.context|raw$, name string, opts $.GetOptions|raw$) (*$.type|raw$, error) { + resp, err := c.client.Get$.ProtoType$(ctx, &$.pb$.Get$.ProtoType$Request{ + Name: name, + Opts: &$.pb$.GetOptions{ + ResourceVersion: opts.ResourceVersion, + Namespace: c.getNamespace(), + }, + }) + if err != nil { + return nil, err + } + + obj := $.FromProto|raw$(resp.Get$.ProtoType$()) + if obj == nil { + return nil, $.fmtErrorf|raw$("received nil $.type|public$ from server for name %s", name) + } + c.logger.V(6).Info("Fetched $.type|public$", + "name", name, + "namespace", c.getNamespace(), + "resource-version", obj.GetResourceVersion(), + ) + + return obj, nil +} +` + +var deleteTemplate = ` +func (c *$.type|allLowercasePlural$) Delete(ctx $.context|raw$, name string, opts $.DeleteOptions|raw$) error { + return $.fmtErrorf|raw$("Delete not implemented for gRPC transport") +} +` + +var createTemplate = ` +func (c *$.type|allLowercasePlural$) Create(ctx $.context|raw$, $.type|allLowercasePlural$ *$.type|raw$, opts $.CreateOptions|raw$) (*$.type|raw$, error) { + return nil, $.fmtErrorf|raw$("Create not implemented for gRPC transport") +} +` + +var updateTemplate = ` +func (c *$.type|allLowercasePlural$) Update(ctx $.context|raw$, $.type|allLowercasePlural$ *$.type|raw$, opts $.UpdateOptions|raw$) (*$.type|raw$, error) { + return nil, $.fmtErrorf|raw$("Update not implemented for gRPC transport") +} +` + +var updateStatusTemplate = ` +func (c *$.type|allLowercasePlural$) UpdateStatus(ctx $.context|raw$, $.type|allLowercasePlural$ *$.type|raw$, opts $.UpdateOptions|raw$) (*$.type|raw$, error) { + return nil, $.fmtErrorf|raw$("UpdateStatus not implemented for gRPC transport") +} +` + +var watchTemplate = ` +func (c *$.type|allLowercasePlural$) Watch(ctx $.context|raw$, opts $.ListOptions|raw$) ($.watchInterface|raw$, error) { + c.logger.V(4).Info("Opening watch stream", + "resource", "$.type|allLowercasePlural$", + "namespace", c.getNamespace(), + "resource-version", opts.ResourceVersion, + ) + + ctx, cancel := context.WithCancel(ctx) + stream, err := c.client.Watch$.ProtoType$s(ctx, &$.pb$.Watch$.ProtoType$sRequest{ + Opts: &$.pb$.ListOptions{ + ResourceVersion: opts.ResourceVersion, + Namespace: c.getNamespace(), + }, + }) + if err != nil { + cancel() + return nil, err + } + + return $.nvgrpc|raw$(&$.type|allLowercasePlural$StreamAdapter{stream: stream}, cancel, c.logger), nil +} +` + +var watchAdapterTemplate = ` +// $.type|allLowercasePlural$StreamAdapter wraps the $.type|public$ gRPC stream to provide events. +type $.type|allLowercasePlural$StreamAdapter struct { + stream $.pb$.$.ProtoType$Service_Watch$.ProtoType$sClient +} + +func (a *$.type|allLowercasePlural$StreamAdapter) Next() (string, $.runtime|raw$, error) { + resp, err := a.stream.Recv() + if err != nil { + return "", nil, err + } + + obj := $.FromProto|raw$(resp.GetObject()) + + return resp.GetType(), obj, nil +} + +func (a *$.type|allLowercasePlural$StreamAdapter) Close() error { + return a.stream.CloseSend() +} +` + +var patchTemplate = ` +func (c *$.type|allLowercasePlural$) Patch(ctx $.context|raw$, name string, pt $.PatchType|raw$, data []byte, opts $.PatchOptions|raw$, subresources ...string) (result *$.type|raw$, err error) { + return nil, $.fmtErrorf|raw$("Patch not implemented for gRPC transport") +} +` diff --git a/code-generator/cmd/client-gen/generators/scheme/generator_for_scheme.go b/code-generator/cmd/client-gen/generators/scheme/generator_for_scheme.go new file mode 100644 index 000000000..c5b57a25d --- /dev/null +++ b/code-generator/cmd/client-gen/generators/scheme/generator_for_scheme.go @@ -0,0 +1,195 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Portions Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved. + +Modified from the original to support gRPC transport. +Origin: https://github.com/kubernetes/code-generator/blob/v0.34.1/cmd/client-gen/generators/scheme/generator_for_scheme.go +*/ + +package scheme + +import ( + "fmt" + "io" + "os" + "path" + "path/filepath" + "strings" + + clientgentypes "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/types" + + "k8s.io/gengo/v2/generator" + "k8s.io/gengo/v2/namer" + "k8s.io/gengo/v2/types" +) + +// GenScheme produces a package for a clientset with the scheme, codecs and parameter codecs. +type GenScheme struct { + generator.GoGenerator + OutputPkg string // Must be a Go import-path + OutputPath string // optional + Groups []clientgentypes.GroupVersions + GroupGoNames map[clientgentypes.GroupVersion]string + InputPackages map[clientgentypes.GroupVersion]string + ImportTracker namer.ImportTracker + PrivateScheme bool + CreateRegistry bool + schemeGenerated bool +} + +func (g *GenScheme) Namers(c *generator.Context) namer.NameSystems { + return namer.NameSystems{ + "raw": namer.NewRawNamer(g.OutputPkg, g.ImportTracker), + } +} + +// We only want to call GenerateType() once. +func (g *GenScheme) Filter(c *generator.Context, t *types.Type) bool { + ret := !g.schemeGenerated + g.schemeGenerated = true + return ret +} + +func (g *GenScheme) Imports(c *generator.Context) (imports []string) { + imports = append(imports, g.ImportTracker.ImportLines()...) + for _, group := range g.Groups { + for _, version := range group.Versions { + packagePath := g.InputPackages[clientgentypes.GroupVersion{Group: group.Group, Version: version.Version}] + groupAlias := strings.ToLower(g.GroupGoNames[clientgentypes.GroupVersion{Group: group.Group, Version: version.Version}]) + if g.CreateRegistry { + // import the install package for internal clientsets instead of the type package with register.go + if version.Version != "" { + packagePath = path.Dir(packagePath) + } + packagePath = path.Join(packagePath, "install") + + imports = append(imports, fmt.Sprintf("%s \"%s\"", groupAlias, packagePath)) + break + } else { + imports = append(imports, fmt.Sprintf("%s%s \"%s\"", groupAlias, strings.ToLower(version.Version.NonEmpty()), packagePath)) + } + } + } + return +} + +func (g *GenScheme) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error { + sw := generator.NewSnippetWriter(w, c, "$", "$") + + allGroupVersions := clientgentypes.ToGroupVersionInfo(g.Groups, g.GroupGoNames) + allInstallGroups := clientgentypes.ToGroupInstallPackages(g.Groups, g.GroupGoNames) + + m := map[string]interface{}{ + "publicScheme": !g.PrivateScheme, + "allGroupVersions": allGroupVersions, + "allInstallGroups": allInstallGroups, + "customRegister": false, + "runtimeNewParameterCodec": c.Universe.Function(types.Name{Package: "k8s.io/apimachinery/pkg/runtime", Name: "NewParameterCodec"}), + "runtimeNewScheme": c.Universe.Function(types.Name{Package: "k8s.io/apimachinery/pkg/runtime", Name: "NewScheme"}), + "serializerNewCodecFactory": c.Universe.Function(types.Name{Package: "k8s.io/apimachinery/pkg/runtime/serializer", Name: "NewCodecFactory"}), + "runtimeScheme": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/runtime", Name: "Scheme"}), + "runtimeSchemeBuilder": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/runtime", Name: "SchemeBuilder"}), + "runtimeUtilMust": c.Universe.Function(types.Name{Package: "k8s.io/apimachinery/pkg/util/runtime", Name: "Must"}), + "schemaGroupVersion": c.Universe.Type(types.Name{Package: "k8s.io/apimachinery/pkg/runtime/schema", Name: "GroupVersion"}), + "metav1AddToGroupVersion": c.Universe.Function(types.Name{Package: "k8s.io/apimachinery/pkg/apis/meta/v1", Name: "AddToGroupVersion"}), + } + globals := map[string]string{ + "Scheme": "Scheme", + "Codecs": "Codecs", + "ParameterCodec": "ParameterCodec", + "Registry": "Registry", + } + for k, v := range globals { + if g.PrivateScheme { + m[k] = strings.ToLower(v[0:1]) + v[1:] + } else { + m[k] = v + } + } + + sw.Do(globalsTemplate, m) + + if g.OutputPath != "" { + if _, err := os.Stat(filepath.Join(g.OutputPath, strings.ToLower("register_custom.go"))); err == nil { + m["customRegister"] = true + } + } + + if g.CreateRegistry { + sw.Do(registryRegistration, m) + } else { + sw.Do(simpleRegistration, m) + } + + return sw.Error() +} + +var globalsTemplate = ` +var $.Scheme$ = $.runtimeNewScheme|raw$() +var $.Codecs$ = $.serializerNewCodecFactory|raw$($.Scheme$) +$if .publicScheme$var $.ParameterCodec$ = $.runtimeNewParameterCodec|raw$($.Scheme$)$end -$` + +var registryRegistration = ` + +func init() { + $.metav1AddToGroupVersion|raw$($.Scheme$, $.schemaGroupVersion|raw${Version: "v1"}) + Install($.Scheme$) +} + +// Install registers the API group and adds types to a scheme +func Install(scheme *$.runtimeScheme|raw$) { + $- range .allInstallGroups$ + $.InstallPackageAlias$.Install(scheme) + $- end$ + $if .customRegister$ + ExtraInstall(scheme) + $end -$ +} +` + +var simpleRegistration = ` +var localSchemeBuilder = $.runtimeSchemeBuilder|raw${ + $- range .allGroupVersions$ + $.PackageAlias$.AddToScheme, + $- end$ + $if .customRegister$ + ExtraAddToScheme, + $end -$ +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + $.metav1AddToGroupVersion|raw$($.Scheme$, $.schemaGroupVersion|raw${Version: "v1"}) + $.runtimeUtilMust|raw$(AddToScheme($.Scheme$)) +} +` diff --git a/code-generator/cmd/client-gen/generators/util/gvpackages.go b/code-generator/cmd/client-gen/generators/util/gvpackages.go new file mode 100644 index 000000000..302693fea --- /dev/null +++ b/code-generator/cmd/client-gen/generators/util/gvpackages.go @@ -0,0 +1,36 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Copied unmodified from the original. +Reason: The upstream package is internal and cannot be imported. +Origin: https://github.com/kubernetes/code-generator/blob/v0.34.1/cmd/client-gen/generators/util/gvpackages.go +*/ + +package util + +import "strings" + +func ParsePathGroupVersion(pgvString string) (gvPath string, gvString string) { + subs := strings.Split(pgvString, "/") + length := len(subs) + switch length { + case 0, 1, 2: + return "", pgvString + default: + return strings.Join(subs[:length-2], "/"), strings.Join(subs[length-2:], "/") + } +} diff --git a/code-generator/cmd/client-gen/generators/util/tags.go b/code-generator/cmd/client-gen/generators/util/tags.go new file mode 100644 index 000000000..a2f805a73 --- /dev/null +++ b/code-generator/cmd/client-gen/generators/util/tags.go @@ -0,0 +1,350 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Copied unmodified from the original. +Reason: The upstream package is internal and cannot be imported. +Origin: https://github.com/kubernetes/code-generator/blob/v0.34.1/cmd/client-gen/generators/util/tags.go +*/ + +package util + +import ( + "errors" + "fmt" + "strings" + + "k8s.io/gengo/v2" +) + +var supportedTags = []string{ + "genclient", + "genclient:nonNamespaced", + "genclient:noVerbs", + "genclient:onlyVerbs", + "genclient:skipVerbs", + "genclient:noStatus", + "genclient:readonly", + "genclient:method", +} + +// SupportedVerbs is a list of supported verbs for +onlyVerbs and +skipVerbs. +var SupportedVerbs = []string{ + "create", + "update", + "updateStatus", + "delete", + "deleteCollection", + "get", + "list", + "watch", + "patch", + "apply", + "applyStatus", +} + +// ReadonlyVerbs represents a list of read-only verbs. +var ReadonlyVerbs = []string{ + "get", + "list", + "watch", +} + +// genClientPrefix is the default prefix for all genclient tags. +const genClientPrefix = "genclient:" + +// unsupportedExtensionVerbs is a list of verbs we don't support generating +// extension client functions for. +var unsupportedExtensionVerbs = []string{ + "updateStatus", + "deleteCollection", + "watch", + "delete", +} + +// inputTypeSupportedVerbs is a list of verb types that supports overriding the +// input argument type. +var inputTypeSupportedVerbs = []string{ + "create", + "update", + "apply", +} + +// resultTypeSupportedVerbs is a list of verb types that supports overriding the +// resulting type. +var resultTypeSupportedVerbs = []string{ + "create", + "update", + "get", + "list", + "patch", + "apply", +} + +// Extensions allows to extend the default set of client verbs +// (CRUD+watch+patch+list+deleteCollection) for a given type with custom defined +// verbs. Custom verbs can have custom input and result types and also allow to +// use a sub-resource in a request instead of top-level resource type. +// +// Example: +// +// +genclient:method=UpdateScale,verb=update,subresource=scale,input=Scale,result=Scale +// +// type ReplicaSet struct { ... } +// +// The 'method=UpdateScale' is the name of the client function. +// The 'verb=update' here means the client function will use 'PUT' action. +// The 'subresource=scale' means we will use SubResource template to generate this client function. +// The 'input' is the input type used for creation (function argument). +// The 'result' (not needed in this case) is the result type returned from the +// client function. +type extension struct { + // VerbName is the name of the custom verb (Scale, Instantiate, etc..) + VerbName string + // VerbType is the type of the verb (only verbs from SupportedVerbs are + // supported) + VerbType string + // SubResourcePath defines a path to a sub-resource to use in the request. + // (optional) + SubResourcePath string + // InputTypeOverride overrides the input parameter type for the verb. By + // default the original type is used. Overriding the input type only works for + // "create" and "update" verb types. The given type must exists in the same + // package as the original type. + // (optional) + InputTypeOverride string + // ResultTypeOverride overrides the resulting object type for the verb. By + // default the original type is used. Overriding the result type works. + // (optional) + ResultTypeOverride string +} + +// IsSubresource indicates if this extension should generate the sub-resource. +func (e *extension) IsSubresource() bool { + return len(e.SubResourcePath) > 0 +} + +// HasVerb checks if the extension matches the given verb. +func (e *extension) HasVerb(verb string) bool { + return e.VerbType == verb +} + +// Input returns the input override package path and the type. +func (e *extension) Input() (string, string) { + parts := strings.Split(e.InputTypeOverride, ".") + return parts[len(parts)-1], strings.Join(parts[0:len(parts)-1], ".") +} + +// Result returns the result override package path and the type. +func (e *extension) Result() (string, string) { + parts := strings.Split(e.ResultTypeOverride, ".") + return parts[len(parts)-1], strings.Join(parts[0:len(parts)-1], ".") +} + +// Tags represents a genclient configuration for a single type. +type Tags struct { + // +genclient + GenerateClient bool + // +genclient:nonNamespaced + NonNamespaced bool + // +genclient:noStatus + NoStatus bool + // +genclient:noVerbs + NoVerbs bool + // +genclient:skipVerbs=get,update + // +genclient:onlyVerbs=create,delete + SkipVerbs []string + // +genclient:method=UpdateScale,verb=update,subresource=scale,input=Scale,result=Scale + Extensions []extension +} + +// HasVerb returns true if we should include the given verb in final client interface and +// generate the function for it. +func (t Tags) HasVerb(verb string) bool { + if len(t.SkipVerbs) == 0 { + return true + } + for _, s := range t.SkipVerbs { + if verb == s { + return false + } + } + return true +} + +// MustParseClientGenTags calls ParseClientGenTags but instead of returning error it panics. +func MustParseClientGenTags(lines []string) Tags { + tags, err := ParseClientGenTags(lines) + if err != nil { + panic(err.Error()) + } + return tags +} + +// ParseClientGenTags parse the provided genclient tags and validates that no unknown +// tags are provided. +func ParseClientGenTags(lines []string) (Tags, error) { + ret := Tags{} + values := gengo.ExtractCommentTags("+", lines) + var value []string + value, ret.GenerateClient = values["genclient"] + // Check the old format and error when used to avoid generating client when //+genclient=false + if len(value) > 0 && len(value[0]) > 0 { + return ret, fmt.Errorf("+genclient=%s is invalid, use //+genclient if you want to generate client or omit it when you want to disable generation", value) + } + _, ret.NonNamespaced = values[genClientPrefix+"nonNamespaced"] + // Check the old format and error when used + if value := values["nonNamespaced"]; len(value) > 0 && len(value[0]) > 0 { + return ret, fmt.Errorf("+nonNamespaced=%s is invalid, use //+genclient:nonNamespaced instead", value[0]) + } + _, ret.NoVerbs = values[genClientPrefix+"noVerbs"] + _, ret.NoStatus = values[genClientPrefix+"noStatus"] + onlyVerbs := []string{} + if _, isReadonly := values[genClientPrefix+"readonly"]; isReadonly { + onlyVerbs = ReadonlyVerbs + } + // Check the old format and error when used + if value := values["readonly"]; len(value) > 0 && len(value[0]) > 0 { + return ret, fmt.Errorf("+readonly=%s is invalid, use //+genclient:readonly instead", value[0]) + } + if v, exists := values[genClientPrefix+"skipVerbs"]; exists { + ret.SkipVerbs = strings.Split(v[0], ",") + } + if v, exists := values[genClientPrefix+"onlyVerbs"]; exists || len(onlyVerbs) > 0 { + if len(v) > 0 { + onlyVerbs = append(onlyVerbs, strings.Split(v[0], ",")...) + } + skipVerbs := []string{} + for _, m := range SupportedVerbs { + skip := true + for _, o := range onlyVerbs { + if o == m { + skip = false + break + } + } + // Check for conflicts + for _, v := range skipVerbs { + if v == m { + return ret, fmt.Errorf("verb %q used both in genclient:skipVerbs and genclient:onlyVerbs", v) + } + } + if skip { + skipVerbs = append(skipVerbs, m) + } + } + ret.SkipVerbs = skipVerbs + } + var err error + if ret.Extensions, err = parseClientExtensions(values); err != nil { + return ret, err + } + return ret, validateClientGenTags(values) +} + +func parseClientExtensions(tags map[string][]string) ([]extension, error) { + var ret []extension + for name, values := range tags { + if !strings.HasPrefix(name, genClientPrefix+"method") { + continue + } + for _, value := range values { + // the value comes in this form: "Foo,verb=create" + ext := extension{} + parts := strings.Split(value, ",") + if len(parts) == 0 { + return nil, fmt.Errorf("invalid or empty extension verb name: %q", value) + } + // The first part represents the name of the extension + ext.VerbName = parts[0] + if len(ext.VerbName) == 0 { + return nil, fmt.Errorf("must specify a verb name (// +genclient:method=Foo,verb=create)") + } + // Parse rest of the arguments + params := parts[1:] + for _, p := range params { + parts := strings.Split(p, "=") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid extension tag specification %q", p) + } + key, val := strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]) + if len(val) == 0 { + return nil, fmt.Errorf("empty value of %q for %q extension", key, ext.VerbName) + } + switch key { + case "verb": + ext.VerbType = val + case "subresource": + ext.SubResourcePath = val + case "input": + ext.InputTypeOverride = val + case "result": + ext.ResultTypeOverride = val + default: + return nil, fmt.Errorf("unknown extension configuration key %q", key) + } + } + // Validate resulting extension configuration + if len(ext.VerbType) == 0 { + return nil, fmt.Errorf("verb type must be specified (use '// +genclient:method=%s,verb=create')", ext.VerbName) + } + if len(ext.ResultTypeOverride) > 0 { + supported := false + for _, v := range resultTypeSupportedVerbs { + if ext.VerbType == v { + supported = true + break + } + } + if !supported { + return nil, fmt.Errorf("%s: result type is not supported for %q verbs (supported verbs: %#v)", ext.VerbName, ext.VerbType, resultTypeSupportedVerbs) + } + } + if len(ext.InputTypeOverride) > 0 { + supported := false + for _, v := range inputTypeSupportedVerbs { + if ext.VerbType == v { + supported = true + break + } + } + if !supported { + return nil, fmt.Errorf("%s: input type is not supported for %q verbs (supported verbs: %#v)", ext.VerbName, ext.VerbType, inputTypeSupportedVerbs) + } + } + for _, t := range unsupportedExtensionVerbs { + if ext.VerbType == t { + return nil, fmt.Errorf("verb %q is not supported by extension generator", ext.VerbType) + } + } + ret = append(ret, ext) + } + } + return ret, nil +} + +// validateTags validates that only supported genclient tags were provided. +func validateClientGenTags(values map[string][]string) error { + for _, k := range supportedTags { + delete(values, k) + } + for key := range values { + if strings.HasPrefix(key, strings.TrimSuffix(genClientPrefix, ":")) { + return errors.New("unknown tag detected: " + key) + } + } + return nil +} diff --git a/code-generator/cmd/client-gen/generators/util/tags_test.go b/code-generator/cmd/client-gen/generators/util/tags_test.go new file mode 100644 index 000000000..4a433e639 --- /dev/null +++ b/code-generator/cmd/client-gen/generators/util/tags_test.go @@ -0,0 +1,154 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Copied unmodified from the original. +Reason: The upstream package is internal and cannot be imported. +Origin: https://github.com/kubernetes/code-generator/blob/v0.34.1/cmd/client-gen/generators/util/tags_test.go +*/ + +package util + +import ( + "reflect" + "testing" +) + +func TestParseTags(t *testing.T) { + testCases := map[string]struct { + lines []string + expectTags Tags + expectError bool + }{ + "genclient": { + lines: []string{`+genclient`}, + expectTags: Tags{GenerateClient: true}, + }, + "genclient=true": { + lines: []string{`+genclient=true`}, + expectError: true, + }, + "nonNamespaced=true": { + lines: []string{`+genclient=true`, `+nonNamespaced=true`}, + expectError: true, + }, + "readonly=true": { + lines: []string{`+genclient=true`, `+readonly=true`}, + expectError: true, + }, + "genclient:nonNamespaced": { + lines: []string{`+genclient`, `+genclient:nonNamespaced`}, + expectTags: Tags{GenerateClient: true, NonNamespaced: true}, + }, + "genclient:noVerbs": { + lines: []string{`+genclient`, `+genclient:noVerbs`}, + expectTags: Tags{GenerateClient: true, NoVerbs: true}, + }, + "genclient:noStatus": { + lines: []string{`+genclient`, `+genclient:noStatus`}, + expectTags: Tags{GenerateClient: true, NoStatus: true}, + }, + "genclient:onlyVerbs": { + lines: []string{`+genclient`, `+genclient:onlyVerbs=create,delete`}, + expectTags: Tags{GenerateClient: true, SkipVerbs: []string{"update", "updateStatus", "deleteCollection", "get", "list", "watch", "patch", "apply", "applyStatus"}}, + }, + "genclient:readonly": { + lines: []string{`+genclient`, `+genclient:readonly`}, + expectTags: Tags{GenerateClient: true, SkipVerbs: []string{"create", "update", "updateStatus", "delete", "deleteCollection", "patch", "apply", "applyStatus"}}, + }, + "genclient:conflict": { + lines: []string{`+genclient`, `+genclient:onlyVerbs=create`, `+genclient:skipVerbs=create`}, + expectError: true, + }, + "genclient:invalid": { + lines: []string{`+genclient`, `+genclient:invalid`}, + expectError: true, + }, + } + for key, c := range testCases { + result, err := ParseClientGenTags(c.lines) + if err != nil && !c.expectError { + t.Fatalf("unexpected error: %v", err) + } + if !c.expectError && !reflect.DeepEqual(result, c.expectTags) { + t.Errorf("[%s] expected %#v to be %#v", key, result, c.expectTags) + } + } +} + +func TestParseTagsExtension(t *testing.T) { + testCases := map[string]struct { + lines []string + expectedExtensions []extension + expectError bool + }{ + "simplest extension": { + lines: []string{`+genclient:method=Foo,verb=create`}, + expectedExtensions: []extension{{VerbName: "Foo", VerbType: "create"}}, + }, + "multiple extensions": { + lines: []string{`+genclient:method=Foo,verb=create`, `+genclient:method=Bar,verb=get`}, + expectedExtensions: []extension{{VerbName: "Foo", VerbType: "create"}, {VerbName: "Bar", VerbType: "get"}}, + }, + "extension without verb": { + lines: []string{`+genclient:method`}, + expectError: true, + }, + "extension without verb type": { + lines: []string{`+genclient:method=Foo`}, + expectError: true, + }, + "sub-resource extension": { + lines: []string{`+genclient:method=Foo,verb=create,subresource=bar`}, + expectedExtensions: []extension{{VerbName: "Foo", VerbType: "create", SubResourcePath: "bar"}}, + }, + "output type extension": { + lines: []string{`+genclient:method=Foos,verb=list,result=Bars`}, + expectedExtensions: []extension{{VerbName: "Foos", VerbType: "list", ResultTypeOverride: "Bars"}}, + }, + "input type extension": { + lines: []string{`+genclient:method=Foo,verb=update,input=Bar`}, + expectedExtensions: []extension{{VerbName: "Foo", VerbType: "update", InputTypeOverride: "Bar"}}, + }, + "unknown verb type extension": { + lines: []string{`+genclient:method=Foo,verb=explode`}, + expectedExtensions: nil, + expectError: true, + }, + "invalid verb extension": { + lines: []string{`+genclient:method=Foo,unknown=bar`}, + expectedExtensions: nil, + expectError: true, + }, + "empty verb extension subresource": { + lines: []string{`+genclient:method=Foo,verb=get,subresource=`}, + expectedExtensions: nil, + expectError: true, + }, + } + for key, c := range testCases { + result, err := ParseClientGenTags(c.lines) + if err != nil && !c.expectError { + t.Fatalf("[%s] unexpected error: %v", key, err) + } + if err != nil && c.expectError { + t.Logf("[%s] got expected error: %+v", key, err) + } + if !c.expectError && !reflect.DeepEqual(result.Extensions, c.expectedExtensions) { + t.Errorf("[%s] expected %#+v to be %#+v", key, result.Extensions, c.expectedExtensions) + } + } +} diff --git a/code-generator/cmd/client-gen/main.go b/code-generator/cmd/client-gen/main.go new file mode 100644 index 000000000..320af1cbc --- /dev/null +++ b/code-generator/cmd/client-gen/main.go @@ -0,0 +1,78 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Portions Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved. + +Modified from the original to support gRPC transport. +Origin: https://github.com/kubernetes/code-generator/blob/v0.34.1/cmd/client-gen/main.go +*/ + +// client-gen makes the individual typed clients using gengo. +package main + +import ( + "flag" + "slices" + + "github.com/spf13/pflag" + "k8s.io/klog/v2" + + "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/args" + "github.com/nvidia/nvsentinel/code-generator/cmd/client-gen/generators" + + "k8s.io/code-generator/pkg/util" + "k8s.io/gengo/v2" + "k8s.io/gengo/v2/generator" +) + +func main() { + klog.InitFlags(nil) + args := args.New() + + args.AddFlags(pflag.CommandLine, "k8s.io/kubernetes/pkg/apis") // TODO: move this input path out of client-gen + flag.Set("logtostderr", "true") + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + pflag.Parse() + + // add group version package as input dirs for gengo + inputPkgs := []string{} + for _, pkg := range args.Groups { + for _, v := range pkg.Versions { + inputPkgs = append(inputPkgs, v.Package) + } + } + // ensure stable code generation output + slices.Sort(inputPkgs) + + if err := args.Validate(); err != nil { + klog.Fatalf("Error: %v", err) + } + + myTargets := func(context *generator.Context) []generator.Target { + return generators.GetTargets(context, args) + } + + if err := gengo.Execute( + generators.NameSystems(util.PluralExceptionListToMapOrDie(args.PluralExceptions)), + generators.DefaultNameSystem(), + myTargets, + gengo.StdBuildTag, + inputPkgs, + ); err != nil { + klog.Fatalf("Error: %v", err) + } +} diff --git a/code-generator/cmd/client-gen/types/helpers.go b/code-generator/cmd/client-gen/types/helpers.go new file mode 100644 index 000000000..08ea0cda2 --- /dev/null +++ b/code-generator/cmd/client-gen/types/helpers.go @@ -0,0 +1,127 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Copied unmodified from the original. +Reason: The upstream package is internal and cannot be imported. +Origin: https://github.com/kubernetes/code-generator/blob/v0.34.1/cmd/client-gen/types/helpers.go +*/ + +package types + +import ( + "fmt" + "regexp" + "sort" + "strings" + + "k8s.io/gengo/v2/namer" +) + +// ToGroupVersion turns "group/version" string into a GroupVersion struct. It reports error +// if it cannot parse the string. +func ToGroupVersion(gv string) (GroupVersion, error) { + // this can be the internal version for the legacy kube types + // TODO once we've cleared the last uses as strings, this special case should be removed. + if (len(gv) == 0) || (gv == "/") { + return GroupVersion{}, nil + } + + switch strings.Count(gv, "/") { + case 0: + return GroupVersion{Group(gv), ""}, nil + case 1: + i := strings.Index(gv, "/") + return GroupVersion{Group(gv[:i]), Version(gv[i+1:])}, nil + default: + return GroupVersion{}, fmt.Errorf("unexpected GroupVersion string: %v", gv) + } +} + +type sortableSliceOfVersions []string + +func (a sortableSliceOfVersions) Len() int { return len(a) } +func (a sortableSliceOfVersions) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a sortableSliceOfVersions) Less(i, j int) bool { + vi, vj := strings.TrimLeft(a[i], "v"), strings.TrimLeft(a[j], "v") + major := regexp.MustCompile("^[0-9]+") + viMajor, vjMajor := major.FindString(vi), major.FindString(vj) + viRemaining, vjRemaining := strings.TrimLeft(vi, viMajor), strings.TrimLeft(vj, vjMajor) + switch { + case len(viRemaining) == 0 && len(vjRemaining) == 0: + return viMajor < vjMajor + case len(viRemaining) == 0 && len(vjRemaining) != 0: + // stable version is greater than unstable version + return false + case len(viRemaining) != 0 && len(vjRemaining) == 0: + // stable version is greater than unstable version + return true + } + // neither are stable versions + if viMajor != vjMajor { + return viMajor < vjMajor + } + // assuming at most we have one alpha or one beta version, so if vi contains "alpha", it's the lesser one. + return strings.Contains(viRemaining, "alpha") +} + +// Determine the default version among versions. If a user calls a group client +// without specifying the version (e.g., c.CoreV1(), instead of c.CoreV1()), the +// default version will be returned. +func defaultVersion(versions []PackageVersion) Version { + var versionStrings []string + for _, version := range versions { + versionStrings = append(versionStrings, version.Version.String()) + } + sort.Sort(sortableSliceOfVersions(versionStrings)) + return Version(versionStrings[len(versionStrings)-1]) +} + +// ToGroupVersionInfo is a helper function used by generators for groups. +func ToGroupVersionInfo(groups []GroupVersions, groupGoNames map[GroupVersion]string) []GroupVersionInfo { + var groupVersionPackages []GroupVersionInfo + for _, group := range groups { + for _, version := range group.Versions { + groupGoName := groupGoNames[GroupVersion{Group: group.Group, Version: version.Version}] + groupVersionPackages = append(groupVersionPackages, GroupVersionInfo{ + Group: Group(namer.IC(group.Group.NonEmpty())), + Version: Version(namer.IC(version.Version.String())), + PackageAlias: strings.ToLower(groupGoName + version.Version.NonEmpty()), + GroupGoName: groupGoName, + LowerCaseGroupGoName: namer.IL(groupGoName), + }) + } + } + return groupVersionPackages +} + +func ToGroupInstallPackages(groups []GroupVersions, groupGoNames map[GroupVersion]string) []GroupInstallPackage { + var groupInstallPackages []GroupInstallPackage + for _, group := range groups { + defaultVersion := defaultVersion(group.Versions) + groupGoName := groupGoNames[GroupVersion{Group: group.Group, Version: defaultVersion}] + groupInstallPackages = append(groupInstallPackages, GroupInstallPackage{ + Group: Group(namer.IC(group.Group.NonEmpty())), + InstallPackageAlias: strings.ToLower(groupGoName), + }) + } + return groupInstallPackages +} + +// NormalizeGroupVersion calls normalizes the GroupVersion. +// func NormalizeGroupVersion(gv GroupVersion) GroupVersion { +// return GroupVersion{Group: gv.Group.NonEmpty(), Version: gv.Version, NonEmptyVersion: normalization.Version(gv.Version)} +// } diff --git a/code-generator/cmd/client-gen/types/helpers_test.go b/code-generator/cmd/client-gen/types/helpers_test.go new file mode 100644 index 000000000..3917b1d60 --- /dev/null +++ b/code-generator/cmd/client-gen/types/helpers_test.go @@ -0,0 +1,38 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Copied unmodified from the original. +Reason: The upstream package is internal and cannot be imported. +Origin: https://github.com/kubernetes/code-generator/blob/v0.34.1/cmd/client-gen/types/helpers_test.go +*/ + +package types + +import ( + "reflect" + "sort" + "testing" +) + +func TestVersionSort(t *testing.T) { + unsortedVersions := []string{"v4beta1", "v2beta1", "v2alpha1", "v3", "v1"} + expected := []string{"v2alpha1", "v2beta1", "v4beta1", "v1", "v3"} + sort.Sort(sortableSliceOfVersions(unsortedVersions)) + if !reflect.DeepEqual(unsortedVersions, expected) { + t.Errorf("expected %#v\ngot %#v", expected, unsortedVersions) + } +} diff --git a/code-generator/cmd/client-gen/types/types.go b/code-generator/cmd/client-gen/types/types.go new file mode 100644 index 000000000..20e8c5708 --- /dev/null +++ b/code-generator/cmd/client-gen/types/types.go @@ -0,0 +1,141 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Portions Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved. + +Modified from the original to support gRPC transport. +Origin: https://github.com/kubernetes/code-generator/blob/v0.34.1/cmd/client-gen/types/types.go +*/ + +package types + +import ( + "fmt" + "path" + "strings" +) + +type Version string + +func (v Version) String() string { + return string(v) +} + +func (v Version) NonEmpty() string { + if v == "" { + return "internalVersion" + } + return v.String() +} + +func (v Version) PackageName() string { + return strings.ToLower(v.NonEmpty()) +} + +type Group string + +func (g Group) String() string { + return string(g) +} + +func (g Group) NonEmpty() string { + if g == "" { + return "core" + } + return string(g) +} + +func (g Group) PackageName() string { + parts := strings.Split(g.NonEmpty(), ".") + if parts[0] == "internal" && len(parts) > 1 { + return strings.ToLower(parts[1] + parts[0]) + } + return strings.ToLower(parts[0]) +} + +type Kind string + +type PackageVersion struct { + Version + // The fully qualified package, e.g. k8s.io/kubernetes/pkg/apis/apps, where the types.go is found. + Package string +} + +type GroupVersion struct { + Group Group + Version Version +} + +type GroupVersionKind struct { + Group Group + Version Version + Kind Kind +} + +func (gv GroupVersion) ToAPIVersion() string { + if len(gv.Group) > 0 && gv.Group != "" { + return gv.Group.String() + "/" + gv.Version.String() + } else { + return gv.Version.String() + } +} + +func (gv GroupVersion) WithKind(kind Kind) GroupVersionKind { + return GroupVersionKind{Group: gv.Group, Version: gv.Version, Kind: kind} +} + +type GroupVersions struct { + // The name of the package for this group, e.g. apps. + PackageName string + Group Group + Versions []PackageVersion +} + +// GroupVersionInfo contains all the info around a group version. +type GroupVersionInfo struct { + Group Group + Version Version + PackageAlias string + GroupGoName string + LowerCaseGroupGoName string +} + +type GroupInstallPackage struct { + Group Group + InstallPackageAlias string +} + +// ProtobufPackage contains the Go package containing protobuf stubs. +type ProtobufPackage struct { + Alias string + Package string // The Go import-path +} + +func NewProtobufPackage(protoBase string, groupPkgName string, version string) ProtobufPackage { + return ProtobufPackage{ + Alias: "pb", + Package: path.Join(protoBase, groupPkgName, version), + } +} + +func (p ProtobufPackage) ImportLines() []string { + return []string{fmt.Sprintf("%s \"%s\"", p.Alias, p.Package)} +} + +func (p ProtobufPackage) ServiceClientConstructorFor(protoType string) string { + return fmt.Sprintf("%s.New%sServiceClient", p.Alias, protoType) +} diff --git a/code-generator/go.mod b/code-generator/go.mod new file mode 100644 index 000000000..0355c270b --- /dev/null +++ b/code-generator/go.mod @@ -0,0 +1,18 @@ +module github.com/nvidia/nvsentinel/code-generator + +go 1.25.5 + +require ( + github.com/spf13/pflag v1.0.10 + golang.org/x/text v0.23.0 + k8s.io/code-generator v0.34.1 + k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b + k8s.io/klog/v2 v2.130.1 +) + +require ( + github.com/go-logr/logr v1.4.3 // indirect + golang.org/x/mod v0.29.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/tools v0.38.0 // indirect +) diff --git a/code-generator/go.sum b/code-generator/go.sum new file mode 100644 index 000000000..da273e03a --- /dev/null +++ b/code-generator/go.sum @@ -0,0 +1,22 @@ +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4= +k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/code-generator v0.34.1 h1:WpphT26E+j7tEgIUfFr5WfbJrktCGzB3JoJH9149xYc= +k8s.io/code-generator v0.34.1/go.mod h1:DeWjekbDnJWRwpw3s0Jat87c+e0TgkxoR4ar608yqvg= +k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b h1:gMplByicHV/TJBizHd9aVEsTYoJBnnUAT5MHlTkbjhQ= +k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b/go.mod h1:CgujABENc3KuTrcsdpGmrrASjtQsWCT7R99mEV4U/fM= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= diff --git a/code-generator/kube_codegen.sh b/code-generator/kube_codegen.sh new file mode 100755 index 000000000..6e845c2e7 --- /dev/null +++ b/code-generator/kube_codegen.sh @@ -0,0 +1,737 @@ +#!/usr/bin/env bash + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Portions Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved. +# +# Modified from the original to support gRPC transport. +# Origin: https://github.com/kubernetes/code-generator/blob/v0.34.1/kube_codegen.sh + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_CODEGEN_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" + +function kube::codegen::internal::get_goversion() { + version=$(grep '^go ' "${KUBE_CODEGEN_ROOT}/go.mod" | awk '{print $2}') + version=${version:-1.25} + echo "$version" +} + +function kube::codegen::internal::get_version() { + local key="$1" + local versions_file="${KUBE_CODEGEN_ROOT}/../.versions.yaml" + if [[ -f "${versions_file}" ]]; then + grep "${key}:" "${versions_file}" | sed -E 's/.*: *//' | tr -d " \"'" || true + fi +} + +if [[ -z "${KUBE_CODEGEN_TAG:-}" ]]; then + if version=$(kube::codegen::internal::get_version "kubernetes_code_gen"); then + KUBE_CODEGEN_TAG="${version}" + fi +fi + +# Callers which want a specific tag of the k8s.io/code-generator repo should +# set the KUBE_CODEGEN_TAG to the tag name, e.g. KUBE_CODEGEN_TAG="release-1.32" +# before sourcing this file. +CODEGEN_VERSION_SPEC="${KUBE_CODEGEN_TAG:+"@${KUBE_CODEGEN_TAG}"}" + +if [[ -z "${PROTOC_GEN_GO_TAG:-}" ]]; then + if version=$(kube::codegen::internal::get_version "protoc_gen_go"); then + PROTOC_GEN_GO_TAG="${version}" + fi +fi + +# Callers which want a specific tag of the google.golang.org/protobuf repo should +# set the PROTOC_GEN_GO_TAG to the tag name, e.g. PROTOC_GEN_GO_TAG="v1.36.10" +# before sourcing this file. +PROTOC_GEN_GO_VERSION_SPEC="${PROTOC_GEN_GO_TAG:+"@${PROTOC_GEN_GO_TAG}"}" + +if [[ -z "${PROTOC_GEN_GO_GRPC_TAG:-}" ]]; then + if version=$(kube::codegen::internal::get_version "protoc_gen_go_grpc"); then + PROTOC_GEN_GO_GRPC_TAG="${version}" + fi +fi + +# Callers which want a specific tag of the google.golang.org/grpc repo should +# set the PROTOC_GEN_GO_GRPC_TAG to the tag name, e.g. PROTOC_GEN_GO_GRPC_TAG="v1.5.1" +# before sourcing this file. +PROTOC_GEN_GO_GRPC_VERSION_SPEC="${PROTOC_GEN_GO_GRPC_TAG:+"@${PROTOC_GEN_GO_GRPC_TAG}"}" + +if [[ -z "${GOVERTER_TAG:-}" ]]; then + if version=$(kube::codegen::internal::get_version "goverter"); then + GOVERTER_TAG="${version}" + fi +fi + +# Callers which want a specific tag of the x repo should +# set the GOVERTER_TAG to the tag name, e.g. GOVERTER_TAG="v1.9.2" +# before sourcing this file. +GOVERTER_VERSION_SPEC="${GOVERTER_TAG:+"@${GOVERTER_TAG}"}" + +# Go installs in $GOBIN if defined, and $GOPATH/bin otherwise. We want to know +# which one it is, so we can use it later. +function get_gobin() { + local from_env + from_env="$(go env GOBIN)" + if [[ -n "${from_env}" ]]; then + echo "${from_env}" + else + echo "$(go env GOPATH)/bin" + fi +} +GOBIN="$(get_gobin)" +export GOBIN + +function kube::codegen::internal::findz() { + # We use `find` rather than `git ls-files` because sometimes external + # projects use this across repos. This is an imperfect wrapper of find, + # but good enough for this script. + find "$@" -print0 +} + +function kube::codegen::internal::grep() { + # We use `grep` rather than `git grep` because sometimes external projects + # use this across repos. + grep "$@" \ + --exclude-dir .git \ + --exclude-dir _output \ + --exclude-dir vendor +} + +# Generate protobuf bindings +# +# USAGE: kube::codegen::gen_proto_bindings [FLAGS] +# +# +# The root directory under which to search for Protobuf files to generate +# bindings for. This must be a local path, not a Go package. +# +# FLAGS: +# +# --output-dir +# The relative path under which to emit code. +# +# --proto-root +# The relative path under which to search for Protobuf definitions. +function kube::codegen::gen_proto_bindings(){ + local in_dir="" + local out_dir="gen/go" + local proto_root="proto" + local v="${KUBE_VERBOSE:-0}" + + while [ "$#" -gt 0 ]; do + case "$1" in + "--output-dir") + out_dir="$2" + shift 2 + ;; + "--proto-root") + proto_root="$2" + shift 2 + ;; + *) + if [[ "$1" =~ ^-- ]]; then + echo "unknown argument: $1" >&2 + return 1 + fi + if [ -n "$in_dir" ]; then + echo "too many arguments: $1 (already have $in_dir)" >&2 + return 1 + fi + in_dir="$1" + shift + ;; + esac + done + + if [ -z "${in_dir}" ]; then + echo "input-dir argument is required" >&2 + return 1 + fi + + ( + # To support running this from anywhere, first cd into this directory, + # and then install with forced module mode on and fully qualified name. + cd "${KUBE_CODEGEN_ROOT}" + UPSTREAM_BINS=( + google.golang.org/protobuf/cmd/protoc-gen-go"${PROTOC_GEN_GO_VERSION_SPEC}" + google.golang.org/grpc/cmd/protoc-gen-go-grpc"${PROTOC_GEN_GO_GRPC_VERSION_SPEC}" + ) + echo "Installing upstream generators..." + for bin in "${UPSTREAM_BINS[@]}"; do + echo " - ${bin}" + GO111MODULE=on go install "${bin}" + done + ) + + # Go bindings + # + local input_versions=() + while read -r dir; do + local version="${dir#"${in_dir}/${proto_root}/"}" + input_versions+=("${version}") + done < <( + ( kube::codegen::internal::findz \ + "${in_dir}/${proto_root}" \ + -type f \ + -name '*.proto' \ + || true \ + ) | while read -r -d $'\0' F; do dirname "${F}"; done \ + | LC_ALL=C sort -u + ) + + if [ "${#input_versions[@]}" != 0 ]; then + echo "Generating Go protobuf bindings for ${#input_versions[@]} targets" + + for version in "${input_versions[@]}"; do + if [ -d "${in_dir}/${out_dir}/${version}" ]; then + ( kube::codegen::internal::findz \ + "${in_dir}/${out_dir}/${version}" \ + -maxdepth 1 \ + -type f \ + -name '*.pb.go' \ + || true \ + ) | xargs -0 rm -f + fi + done + + ( + cd "${in_dir}/${proto_root}" + for version in "${input_versions[@]}"; do + mkdir -p "../${out_dir}/${version}" + protoc \ + -I . \ + --plugin="protoc-gen-go=${GOBIN}/protoc-gen-go" \ + --plugin="protoc-gen-go-grpc=${GOBIN}/protoc-gen-go-grpc" \ + --go_out="../${out_dir}" \ + --go_opt=paths="source_relative" \ + --go-grpc_out="../${out_dir}" \ + --go-grpc_opt=paths="source_relative" \ + "${version}"/*.proto + done + ) + fi +} + +# Generate tagged helper code: conversions, deepcopy, defaults and validations +# +# USAGE: kube::codegen::gen_helpers [FLAGS] +# +# +# The root directory under which to search for Go files which request code to +# be generated. This must be a local path, not a Go package. +# +# See note at the top about package structure below that. +# +# FLAGS: +# +# --boilerplate +# An optional override for the header file to insert into generated files. +# +# --extra-peer-dir +# An optional list (this flag may be specified multiple times) of "extra" +# directories to consider during conversion generation. +# +function kube::codegen::gen_helpers() { + local in_dir="" + local boilerplate="${KUBE_CODEGEN_ROOT}/hack/boilerplate.go.txt" + local v="${KUBE_VERBOSE:-0}" + local extra_peers=() + + while [ "$#" -gt 0 ]; do + case "$1" in + "--boilerplate") + boilerplate="$2" + shift 2 + ;; + "--extra-peer-dir") + extra_peers+=("$2") + shift 2 + ;; + *) + if [[ "$1" =~ ^-- ]]; then + echo "unknown argument: $1" >&2 + return 1 + fi + if [ -n "$in_dir" ]; then + echo "too many arguments: $1 (already have $in_dir)" >&2 + return 1 + fi + in_dir="$1" + shift + ;; + esac + done + + if [ -z "${in_dir}" ]; then + echo "input-dir argument is required" >&2 + return 1 + fi + + ( + # To support running this from anywhere, first cd into this directory, + # and then install with forced module mode on and fully qualified name. + cd "${KUBE_CODEGEN_ROOT}" + UPSTREAM_BINS=( + "k8s.io/code-generator/cmd/conversion-gen${CODEGEN_VERSION_SPEC}" + "k8s.io/code-generator/cmd/deepcopy-gen${CODEGEN_VERSION_SPEC}" + "k8s.io/code-generator/cmd/defaulter-gen${CODEGEN_VERSION_SPEC}" + "k8s.io/code-generator/cmd/validation-gen${CODEGEN_VERSION_SPEC}" + ) + echo "Installing upstream generators..." + for bin in "${UPSTREAM_BINS[@]}"; do + echo " - ${bin}" + done + # shellcheck disable=2046 # printf word-splitting is intentional + GO111MODULE=on go install -a $(printf "%s " "${UPSTREAM_BINS[@]}") + + echo "Installing goverter..." + rm -f "${GOBIN}/goverter" + local tmp_dir + tmp_dir=$(mktemp -d) + trap 'rm -rf -- "$tmp_dir"' EXIT + + local goversion + goversion=$(kube::codegen::internal::get_goversion) + + pushd "${tmp_dir}" > /dev/null + go mod init build-goverter > /dev/null 2>&1 + go mod edit -go="${goversion}" > /dev/null 2>&1 + export GOTOOLCHAIN=auto + go get "github.com/jmattheis/goverter${GOVERTER_VERSION_SPEC}" > /dev/null 2>&1 + go build -o "${GOBIN}/goverter" "github.com/jmattheis/goverter/cmd/goverter" > /dev/null + popd > /dev/null + echo " - github.com/jmattheis/goverter${GOVERTER_VERSION_SPEC}" + ) + + # Deepcopy + # + local input_pkgs=() + while read -r dir; do + pkg="$(cd "${dir}" && GO111MODULE=on go list -find .)" + input_pkgs+=("${pkg}") + done < <( + ( kube::codegen::internal::grep -l --null \ + -e '^\s*//\s*+k8s:deepcopy-gen=' \ + -r "${in_dir}" \ + --include '*.go' \ + || true \ + ) | while read -r -d $'\0' F; do dirname "${F}"; done \ + | LC_ALL=C sort -u + ) + + if [ "${#input_pkgs[@]}" != 0 ]; then + echo "Generating deepcopy code for ${#input_pkgs[@]} targets" + + kube::codegen::internal::findz \ + "${in_dir}" \ + -type f \ + -name zz_generated.deepcopy.go \ + | xargs -0 rm -f + + "${GOBIN}/deepcopy-gen" \ + -v "${v}" \ + --output-file zz_generated.deepcopy.go \ + --go-header-file "${boilerplate}" \ + "${input_pkgs[@]}" + fi + + # Validations + # + local input_pkgs=() + while read -r dir; do + pkg="$(cd "${dir}" && GO111MODULE=on go list -find .)" + input_pkgs+=("${pkg}") + done < <( + ( kube::codegen::internal::grep -l --null \ + -e '^\s*//\s*+k8s:validation-gen=' \ + -r "${in_dir}" \ + --include '*.go' \ + || true \ + ) | while read -r -d $'\0' F; do dirname "${F}"; done \ + | LC_ALL=C sort -u + ) + + if [ "${#input_pkgs[@]}" != 0 ]; then + echo "Generating validation code for ${#input_pkgs[@]} targets" + + kube::codegen::internal::findz \ + "${in_dir}" \ + -type f \ + -name zz_generated.validations.go \ + | xargs -0 rm -f + + "${GOBIN}/validation-gen" \ + -v "${v}" \ + --output-file zz_generated.validations.go \ + --go-header-file "${boilerplate}" \ + "${input_pkgs[@]}" + fi + + # Defaults + # + local input_pkgs=() + while read -r dir; do + pkg="$(cd "${dir}" && GO111MODULE=on go list -find .)" + input_pkgs+=("${pkg}") + done < <( + ( kube::codegen::internal::grep -l --null \ + -e '^\s*//\s*+k8s:defaulter-gen=' \ + -r "${in_dir}" \ + --include '*.go' \ + || true \ + ) | while read -r -d $'\0' F; do dirname "${F}"; done \ + | LC_ALL=C sort -u + ) + + if [ "${#input_pkgs[@]}" != 0 ]; then + echo "Generating defaulter code for ${#input_pkgs[@]} targets" + + kube::codegen::internal::findz \ + "${in_dir}" \ + -type f \ + -name zz_generated.defaults.go \ + | xargs -0 rm -f + + "${GOBIN}/defaulter-gen" \ + -v "${v}" \ + --output-file zz_generated.defaults.go \ + --go-header-file "${boilerplate}" \ + "${input_pkgs[@]}" + fi + + # Conversions + # + local input_pkgs=() + while read -r dir; do + pkg="$(cd "${dir}" && GO111MODULE=on go list -find .)" + input_pkgs+=("${pkg}") + done < <( + ( kube::codegen::internal::grep -l --null \ + -e '^\s*//\s*+k8s:conversion-gen=' \ + -r "${in_dir}" \ + --include '*.go' \ + || true \ + ) | while read -r -d $'\0' F; do dirname "${F}"; done \ + | LC_ALL=C sort -u + ) + + if [ "${#input_pkgs[@]}" != 0 ]; then + echo "Generating conversion code for ${#input_pkgs[@]} targets" + + kube::codegen::internal::findz \ + "${in_dir}" \ + -type f \ + -name zz_generated.conversion.go \ + | xargs -0 rm -f + + local extra_peer_args=() + for arg in "${extra_peers[@]:+"${extra_peers[@]}"}"; do + extra_peer_args+=("--extra-peer-dirs" "$arg") + done + "${GOBIN}/conversion-gen" \ + -v "${v}" \ + --output-file zz_generated.conversion.go \ + --go-header-file "${boilerplate}" \ + "${extra_peer_args[@]:+"${extra_peer_args[@]}"}" \ + "${input_pkgs[@]}" + fi + + local input_dirs=() + while read -r dir; do + input_dirs+=("${dir}") + done < <( + ( kube::codegen::internal::grep -l --null \ + -e '^\s*//\s*goverter:converter' \ + -r "${in_dir}" \ + --include '*.go' \ + || true \ + ) | while read -r -d $'\0' F; do dirname "${F}"; done \ + | LC_ALL=C sort -u + ) + + if [ "${#input_dirs[@]}" != 0 ]; then + echo "Generating goverter conversion code for ${#input_dirs[@]} targets" + + kube::codegen::internal::findz \ + "${in_dir}" \ + -type f \ + -name zz_generated.goverter.go \ + | xargs -0 rm -f + + for dir in "${input_dirs[@]}"; do + "${GOBIN}/goverter" \ + gen \ + "${dir}" + done + fi +} + +# Generate client code +# +# USAGE: kube::codegen::gen_client [FLAGS] +# +# +# The root package under which to search for Go files which request clients +# to be generated. This must be a local path, not a Go package. +# +# FLAGS: +# --one-input-api +# A specific API (a directory) under the input-dir for which to generate a +# client. If this is not set, clients for all APIs under the input-dir +# will be generated (under the --output-pkg). +# +# --output-dir +# The root directory under which to emit code. Each aspect of client +# generation will make one or more subdirectories. +# +# --output-pkg +# The Go package path (import path) of the --output-dir. Each aspect of +# client generation will make one or more sub-packages. +# +# --boilerplate +# An optional override for the header file to insert into generated files. +# +# --clientset-name +# An optional override for the leaf name of the generated "clientset" directory. +# +# --versioned-name +# An optional override for the leaf name of the generated +# "/versioned" directory. +# +# --with-watch +# Enables generation of listers and informers for APIs which support WATCH. +# +# --listers-name +# An optional override for the leaf name of the generated "listers" directory. +# +# --informers-name +# An optional override for the leaf name of the generated "informers" directory. +# +# --plural-exceptions +# An optional list of comma separated plural exception definitions in Type:PluralizedType form. +# +# --proto-base +# The base Go import-path of the protobuf stubs. +# +function kube::codegen::gen_client() { + local in_dir="" + local one_input_api="" + local out_dir="" + local out_pkg="" + local clientset_subdir="clientset" + local clientset_versioned_name="versioned" + local watchable="false" + local listers_subdir="listers" + local informers_subdir="informers" + local boilerplate="${KUBE_CODEGEN_ROOT}/hack/boilerplate.go.txt" + local plural_exceptions="" + local v="${KUBE_VERBOSE:-0}" + local proto_base="" + + while [ "$#" -gt 0 ]; do + case "$1" in + "--one-input-api") + one_input_api="/$2" + shift 2 + ;; + "--output-dir") + out_dir="$2" + shift 2 + ;; + "--output-pkg") + out_pkg="$2" + shift 2 + ;; + "--boilerplate") + boilerplate="$2" + shift 2 + ;; + "--clientset-name") + clientset_subdir="$2" + shift 2 + ;; + "--versioned-name") + clientset_versioned_name="$2" + shift 2 + ;; + "--with-watch") + watchable="true" + shift + ;; + "--listers-name") + listers_subdir="$2" + shift 2 + ;; + "--informers-name") + informers_subdir="$2" + shift 2 + ;; + "--plural-exceptions") + plural_exceptions="$2" + shift 2 + ;; + "--proto-base") + proto_base="$2" + shift 2 + ;; + *) + if [[ "$1" =~ ^-- ]]; then + echo "unknown argument: $1" >&2 + return 1 + fi + if [ -n "$in_dir" ]; then + echo "too many arguments: $1 (already have $in_dir)" >&2 + return 1 + fi + in_dir="$1" + shift + ;; + esac + done + + if [ -z "${in_dir}" ]; then + echo "input-dir argument is required" >&2 + return 1 + fi + if [ -z "${out_dir}" ]; then + echo "--output-dir is required" >&2 + return 1 + fi + if [ -z "${out_pkg}" ]; then + echo "--output-pkg is required" >&2 + return 1 + fi + if [ -z "${proto_base}" ]; then + echo "--proto-base is required for gRPC client generation" >&2 + return 1 + fi + + mkdir -p "${out_dir}" + + ( + # To support running this from anywhere, first cd into this directory, + # and then install with forced module mode on and fully qualified name. + cd "${KUBE_CODEGEN_ROOT}" + + UPSTREAM_BINS=( + informer-gen"${CODEGEN_VERSION_SPEC}" + lister-gen"${CODEGEN_VERSION_SPEC}" + ) + echo "Installing upstream generators..." + for bin in "${UPSTREAM_BINS[@]}"; do + echo " - k8s.io/code-generator/cmd/${bin}" + done + # shellcheck disable=2046 # printf word-splitting is intentional + GO111MODULE=on go install $(printf "k8s.io/code-generator/cmd/%s " "${UPSTREAM_BINS[@]}") + + echo "Installing local generators..." + rm -f "${GOBIN}/client-gen" + GO111MODULE=on go build -a -o "${GOBIN}/client-gen" ./cmd/client-gen + echo " - github.com/nvidia/nvsentinel/code-generator/cmd/client-gen${CODEGEN_VERSION_SPEC}" + ) + + local group_versions=() + local input_pkgs=() + while read -r dir; do + pkg="$(cd "${dir}" && GO111MODULE=on go list -find .)" + leaf="$(basename "${dir}")" + if grep -E -q '^v[0-9]+((alpha|beta)[0-9]+)?$' <<< "${leaf}"; then + input_pkgs+=("${pkg}") + + dir2="$(dirname "${dir}")" + leaf2="$(basename "${dir2}")" + group_versions+=("${leaf2}/${leaf}") + fi + done < <( + ( kube::codegen::internal::grep -l --null \ + -e '^[[:space:]]*//[[:space:]]*+genclient' \ + -r "${in_dir}${one_input_api}" \ + --include '*.go' \ + || true \ + ) | while read -r -d $'\0' F; do dirname "${F}"; done \ + | LC_ALL=C sort -u + ) + + if [ "${#group_versions[@]}" == 0 ]; then + return 0 + fi + + echo "Generating client code for ${#group_versions[@]} targets" + + ( kube::codegen::internal::grep -l --null \ + -e '^// Code generated by client-gen. DO NOT EDIT.$' \ + -r "${out_dir}/${clientset_subdir}" \ + --include '*.go' \ + || true \ + ) | xargs -0 rm -f + + local inputs=() + for arg in "${group_versions[@]}"; do + inputs+=("--input" "$arg") + done + + "${GOBIN}/client-gen" \ + -v "${v}" \ + --go-header-file "${boilerplate}" \ + --output-dir "${out_dir}/${clientset_subdir}" \ + --output-pkg "${out_pkg}/${clientset_subdir}" \ + --clientset-name "${clientset_versioned_name}" \ + --input-base "$(cd "${in_dir}" && pwd -P)" \ + --plural-exceptions "${plural_exceptions}" \ + --proto-base="${proto_base}" \ + "${inputs[@]}" + + if [ "${watchable}" == "true" ]; then + echo "Generating lister code for ${#input_pkgs[@]} targets" + + ( kube::codegen::internal::grep -l --null \ + -e '^// Code generated by lister-gen. DO NOT EDIT.$' \ + -r "${out_dir}/${listers_subdir}" \ + --include '*.go' \ + || true \ + ) | xargs -0 rm -f + + "${GOBIN}/lister-gen" \ + -v "${v}" \ + --go-header-file "${boilerplate}" \ + --output-dir "${out_dir}/${listers_subdir}" \ + --output-pkg "${out_pkg}/${listers_subdir}" \ + --plural-exceptions "${plural_exceptions}" \ + "${input_pkgs[@]}" + + echo "Generating informer code for ${#input_pkgs[@]} targets" + + ( kube::codegen::internal::grep -l --null \ + -e '^// Code generated by informer-gen. DO NOT EDIT.$' \ + -r "${out_dir}/${informers_subdir}" \ + --include '*.go' \ + || true \ + ) | xargs -0 rm -f + + "${GOBIN}/informer-gen" \ + -v "${v}" \ + --go-header-file "${boilerplate}" \ + --output-dir "${out_dir}/${informers_subdir}" \ + --output-pkg "${out_pkg}/${informers_subdir}" \ + --versioned-clientset-package "${out_pkg}/${clientset_subdir}/${clientset_versioned_name}" \ + --listers-package "${out_pkg}/${listers_subdir}" \ + --plural-exceptions "${plural_exceptions}" \ + "${input_pkgs[@]}" + fi +} diff --git a/demos/nvml-sidecar-demo.sh b/demos/nvml-sidecar-demo.sh new file mode 100755 index 000000000..d7964ad6a --- /dev/null +++ b/demos/nvml-sidecar-demo.sh @@ -0,0 +1,498 @@ +#!/bin/bash +# NVML Provider Sidecar Demo +# Demonstrates the NVML provider sidecar architecture for GPU enumeration +# +# Prerequisites: +# - kubectl configured with GPU cluster access +# - docker or podman for building images +# - helm 3.x installed +# - GPU nodes with RuntimeClass 'nvidia' +# +# Usage: ./demos/nvml-sidecar-demo.sh [kubeconfig] + +set -euo pipefail + +# ============================================================================== +# Configuration +# ============================================================================== + +KUBECONFIG="${1:-$HOME/.kube/config-aws-gpu}" +NAMESPACE="nvsentinel" +RELEASE_NAME="nvsentinel" +CHART_PATH="deployments/helm/nvsentinel" +VALUES_FILE="deployments/helm/values-sidecar-test.yaml" +DOCKERFILE="deployments/container/Dockerfile" + +# Image settings (using ttl.sh ephemeral registry - images expire after 2h) +SERVER_IMAGE="ttl.sh/device-api-server-sidecar:2h" +SIDECAR_IMAGE="ttl.sh/nvml-provider-sidecar:2h" + +# ============================================================================== +# Terminal Colors (buildah-style) +# ============================================================================== + +if [[ -t 1 ]]; then + red=$(tput setaf 1) + green=$(tput setaf 2) + yellow=$(tput setaf 3) + blue=$(tput setaf 4) + magenta=$(tput setaf 5) + cyan=$(tput setaf 6) + white=$(tput setaf 7) + bold=$(tput bold) + reset=$(tput sgr0) +else + red="" + green="" + yellow="" + blue="" + magenta="" + cyan="" + white="" + bold="" + reset="" +fi + +# ============================================================================== +# Helper Functions +# ============================================================================== + +banner() { + echo "" + echo "${bold}${blue}============================================================${reset}" + echo "${bold}${blue} $1${reset}" + echo "${bold}${blue}============================================================${reset}" + echo "" +} + +step() { + echo "" + echo "${bold}${green}>>> $1${reset}" + echo "" +} + +info() { + echo "${cyan} $1${reset}" +} + +warn() { + echo "${yellow} WARNING: $1${reset}" +} + +error() { + echo "${red} ERROR: $1${reset}" +} + +run_cmd() { + echo "${magenta} \$ $*${reset}" + "$@" +} + +pause() { + echo "" + read -r -p "${yellow}Press ENTER to continue...${reset}" + echo "" +} + +confirm() { + echo "" + read -r -p "${yellow}$1 [y/N] ${reset}" response + case "$response" in + [yY][eE][sS]|[yY]) return 0 ;; + *) return 1 ;; + esac +} + +check_prereqs() { + local missing=() + + command -v kubectl &>/dev/null || missing+=("kubectl") + command -v helm &>/dev/null || missing+=("helm") + command -v docker &>/dev/null || missing+=("docker") + + if [[ ${#missing[@]} -gt 0 ]]; then + error "Missing prerequisites: ${missing[*]}" + exit 1 + fi + + # Check for buildx (required for cross-platform builds) + if ! docker buildx version &>/dev/null; then + warn "docker buildx not available - cross-platform builds may fail" + warn "Run: docker buildx create --use --name multiarch" + else + info "Docker buildx: $(docker buildx version | head -1)" + fi +} + +# ============================================================================== +# Demo Sections +# ============================================================================== + +show_intro() { + clear + banner "NVML Provider Sidecar Architecture Demo" + + echo "${white}This demo showcases the new sidecar-based NVML provider for NVSentinel.${reset}" + echo "" + echo "${white}Architecture:${reset}" + echo "${cyan} ┌─────────────────────────────────────────────────────────┐${reset}" + echo "${cyan} │ Pod │${reset}" + echo "${cyan} │ ┌──────────────────┐ ┌──────────────────┐ │${reset}" + echo "${cyan} │ │ device-api-server│ │ nvml-provider │ │${reset}" + echo "${cyan} │ │ (pure Go) │◄───│ (CGO + NVML) │ │${reset}" + echo "${cyan} │ │ Port 8080 │gRPC│ Port 9001 │ │${reset}" + echo "${cyan} │ │ │ │ RuntimeClass: │ │${reset}" + echo "${cyan} │ │ No NVML deps │ │ nvidia │ │${reset}" + echo "${cyan} │ └──────────────────┘ └──────────────────┘ │${reset}" + echo "${cyan} └─────────────────────────────────────────────────────────┘${reset}" + echo "" + echo "${white}Benefits:${reset}" + echo "${green} ✓ Separation of concerns (API server vs NVML access)${reset}" + echo "${green} ✓ Independent scaling and updates${reset}" + echo "${green} ✓ Better testability (mock providers)${reset}" + echo "${green} ✓ Crash isolation (NVML crashes don't kill API server)${reset}" + echo "" + + pause +} + +show_cluster_info() { + banner "Step 1: Verify Cluster Connectivity" + + info "Using kubeconfig: ${KUBECONFIG}" + echo "" + + step "Check cluster connection" + run_cmd kubectl --kubeconfig="${KUBECONFIG}" cluster-info + + pause + + step "List GPU nodes (with node-type=gpu label)" + run_cmd kubectl --kubeconfig="${KUBECONFIG}" get nodes -l node-type=gpu -o wide + + pause + + step "Verify nvidia RuntimeClass exists" + run_cmd kubectl --kubeconfig="${KUBECONFIG}" get runtimeclass nvidia -o yaml || { + warn "RuntimeClass 'nvidia' not found. GPU access may not work." + } + + pause +} + +check_image_exists() { + local image="$1" + # Try to inspect the manifest - if it exists, the image is available + docker buildx imagetools inspect "${image}" &>/dev/null 2>&1 +} + +build_images() { + banner "Step 2: Build Container Images" + + info "Building images for ephemeral registry (ttl.sh - 2 hour expiry)" + info "Using unified multi-target Dockerfile at ${DOCKERFILE}" + info "Target platform: linux/amd64 (cross-compile for x86 clusters)" + echo "" + + # Ensure buildx is available for cross-platform builds + if ! docker buildx version &>/dev/null; then + error "docker buildx is required for cross-platform builds" + error "Install Docker Desktop or run: docker buildx create --use" + exit 1 + fi + + # Check if images already exist + local need_server=true + local need_sidecar=true + + if check_image_exists "${SERVER_IMAGE}"; then + info "Image ${SERVER_IMAGE} already exists" + if ! confirm "Rebuild device-api-server image?"; then + need_server=false + fi + fi + + if check_image_exists "${SIDECAR_IMAGE}"; then + info "Image ${SIDECAR_IMAGE} already exists" + if ! confirm "Rebuild nvml-provider image?"; then + need_sidecar=false + fi + fi + + if [[ "${need_server}" == "true" ]]; then + step "Build and push device-api-server image (CGO_ENABLED=0)" + info "This is a pure Go binary with no NVML dependencies" + info "Building for linux/amd64 and pushing directly..." + run_cmd docker buildx build \ + --platform linux/amd64 \ + --target device-api-server \ + -t "${SERVER_IMAGE}" \ + -f "${DOCKERFILE}" \ + --push \ + . + pause + else + info "Skipping device-api-server build" + fi + + if [[ "${need_sidecar}" == "true" ]]; then + step "Build and push nvml-provider sidecar image (CGO_ENABLED=1)" + info "This requires glibc runtime for NVML library binding" + info "Building for linux/amd64 and pushing directly..." + run_cmd docker buildx build \ + --platform linux/amd64 \ + --target nvml-provider \ + -t "${SIDECAR_IMAGE}" \ + -f "${DOCKERFILE}" \ + --push \ + . + pause + else + info "Skipping nvml-provider build" + fi +} + +show_values_file() { + banner "Step 3: Review Helm Values" + + info "The sidecar architecture is enabled via Helm values" + echo "" + + step "Key configuration in ${VALUES_FILE}:" + echo "" + echo "${cyan}# Disable built-in NVML provider${reset}" + echo "${white}nvml:${reset}" + echo "${white} enabled: false${reset}" + echo "" + echo "${cyan}# Enable NVML Provider sidecar${reset}" + echo "${white}nvmlProvider:${reset}" + echo "${white} enabled: true${reset}" + echo "${white} image:${reset}" + echo "${white} repository: ttl.sh/nvml-provider-sidecar${reset}" + echo "${white} tag: \"2h\"${reset}" + echo "${white} serverAddress: \"localhost:9001\"${reset}" + echo "${white} runtimeClassName: nvidia${reset}" + echo "" + + if [[ -f "${VALUES_FILE}" ]]; then + step "Full values file:" + run_cmd cat "${VALUES_FILE}" + fi + + pause +} + +deploy_sidecar() { + banner "Step 4: Deploy with Sidecar Architecture" + + step "Create namespace if not exists" + echo "${magenta} \$ kubectl create namespace ${NAMESPACE} --dry-run=client -o yaml | kubectl apply -f -${reset}" + kubectl --kubeconfig="${KUBECONFIG}" create namespace "${NAMESPACE}" --dry-run=client -o yaml | \ + kubectl --kubeconfig="${KUBECONFIG}" apply -f - + + pause + + # Check if release already exists + if helm status "${RELEASE_NAME}" --kubeconfig="${KUBECONFIG}" -n "${NAMESPACE}" &>/dev/null; then + info "Release '${RELEASE_NAME}' already exists" + step "Upgrading existing release..." + run_cmd helm upgrade "${RELEASE_NAME}" "${CHART_PATH}" \ + --kubeconfig="${KUBECONFIG}" \ + --namespace "${NAMESPACE}" \ + -f "${VALUES_FILE}" + + step "Restarting pods to pick up changes..." + run_cmd kubectl --kubeconfig="${KUBECONFIG}" -n "${NAMESPACE}" rollout restart daemonset "${RELEASE_NAME}" + else + step "Installing new release..." + run_cmd helm install "${RELEASE_NAME}" "${CHART_PATH}" \ + --kubeconfig="${KUBECONFIG}" \ + --namespace "${NAMESPACE}" \ + -f "${VALUES_FILE}" + fi + + pause + + step "Waiting for pods to be ready (timeout 2m)..." + if ! kubectl --kubeconfig="${KUBECONFIG}" -n "${NAMESPACE}" rollout status daemonset "${RELEASE_NAME}" --timeout=2m; then + warn "Rollout not complete within timeout. Checking status..." + fi + + step "Current pod status" + run_cmd kubectl --kubeconfig="${KUBECONFIG}" -n "${NAMESPACE}" get pods -l app.kubernetes.io/name=nvsentinel -o wide + + pause + + step "Verify both containers are running in each pod" + info "Each pod should have 2/2 containers ready" + run_cmd kubectl --kubeconfig="${KUBECONFIG}" -n "${NAMESPACE}" get pods -l app.kubernetes.io/name=nvsentinel \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.phase}{"\t"}{range .status.containerStatuses[*]}{.name}:{.ready}{" "}{end}{"\n"}{end}' + + pause +} + +verify_gpu_registration() { + banner "Step 5: Verify GPU Registration" + + step "Wait for pods to be ready" + info "Waiting up to 60 seconds for pods to start..." + if ! kubectl --kubeconfig="${KUBECONFIG}" -n "${NAMESPACE}" wait --for=condition=ready pod -l app.kubernetes.io/name=nvsentinel --timeout=60s 2>/dev/null; then + warn "Pods may not be ready yet. Checking status..." + run_cmd kubectl --kubeconfig="${KUBECONFIG}" -n "${NAMESPACE}" get pods -l app.kubernetes.io/name=nvsentinel -o wide + run_cmd kubectl --kubeconfig="${KUBECONFIG}" -n "${NAMESPACE}" describe pods -l app.kubernetes.io/name=nvsentinel | tail -30 + error "Pods not ready. Check the output above for issues." + return 1 + fi + + step "Get a pod name for testing" + POD=$(kubectl --kubeconfig="${KUBECONFIG}" -n "${NAMESPACE}" get pods -l app.kubernetes.io/name=nvsentinel -o jsonpath='{.items[0].metadata.name}') + if [[ -z "${POD}" ]]; then + error "No pods found. DaemonSet may not be scheduling on any nodes." + run_cmd kubectl --kubeconfig="${KUBECONFIG}" -n "${NAMESPACE}" get daemonset + return 1 + fi + info "Using pod: ${POD}" + + pause + + step "Check device-api-server logs for provider connection" + run_cmd kubectl --kubeconfig="${KUBECONFIG}" -n "${NAMESPACE}" logs "${POD}" -c nvsentinel --tail=20 || true + + pause + + step "Check nvml-provider sidecar logs" + run_cmd kubectl --kubeconfig="${KUBECONFIG}" -n "${NAMESPACE}" logs "${POD}" -c nvml-provider --tail=20 || true + + pause +} + +demonstrate_crash_recovery() { + banner "Step 6: Demonstrate Crash Recovery" + + info "The sidecar architecture provides crash isolation." + info "If the NVML provider crashes, the API server continues running" + info "and will reconnect when the provider restarts." + echo "" + + step "Get current pod" + POD=$(kubectl --kubeconfig="${KUBECONFIG}" -n "${NAMESPACE}" get pods -l app.kubernetes.io/name=nvsentinel -o jsonpath='{.items[0].metadata.name}') + info "Using pod: ${POD}" + + pause + + if confirm "Kill the nvml-provider container to demonstrate crash recovery?"; then + step "Killing nvml-provider container..." + run_cmd kubectl --kubeconfig="${KUBECONFIG}" -n "${NAMESPACE}" exec "${POD}" -c nvml-provider -- kill 1 || true + + info "Waiting for container restart..." + sleep 5 + + step "Check pod status (should show restart count)" + run_cmd kubectl --kubeconfig="${KUBECONFIG}" -n "${NAMESPACE}" get pod "${POD}" -o wide + + step "Verify API server continued running" + run_cmd kubectl --kubeconfig="${KUBECONFIG}" -n "${NAMESPACE}" logs "${POD}" -c nvsentinel --tail=10 || true + + step "Verify provider reconnected" + run_cmd kubectl --kubeconfig="${KUBECONFIG}" -n "${NAMESPACE}" logs "${POD}" -c nvml-provider --tail=10 || true + else + info "Skipping crash recovery demonstration" + fi + + pause +} + +show_metrics() { + banner "Step 7: View Provider Metrics" + + step "Get pod for port-forward" + POD=$(kubectl --kubeconfig="${KUBECONFIG}" -n "${NAMESPACE}" get pods -l app.kubernetes.io/name=nvsentinel -o jsonpath='{.items[0].metadata.name}') + + step "Fetch metrics from the API server" + info "Key metrics to look for:" + info " - device_api_provider_connected: Whether provider is connected" + info " - device_api_gpus_total: Number of GPUs registered" + info " - device_api_provider_heartbeat_*: Heartbeat latency" + echo "" + + run_cmd kubectl --kubeconfig="${KUBECONFIG}" -n "${NAMESPACE}" exec "${POD}" -c nvsentinel -- \ + wget -qO- http://localhost:8081/metrics 2>/dev/null | grep -E "^device_api_" | sort || { + run_cmd kubectl --kubeconfig="${KUBECONFIG}" -n "${NAMESPACE}" exec "${POD}" -c nvsentinel -- \ + curl -s http://localhost:8081/metrics 2>/dev/null | grep -E "^device_api_" | sort || true + } + + pause +} + +cleanup() { + banner "Cleanup" + + if confirm "Remove the sidecar deployment and restore default?"; then + step "Uninstalling Helm release..." + run_cmd helm uninstall "${RELEASE_NAME}" \ + --kubeconfig="${KUBECONFIG}" \ + --namespace "${NAMESPACE}" || true + + info "Cleanup complete!" + else + info "Skipping cleanup. Release '${RELEASE_NAME}' left in namespace '${NAMESPACE}'" + fi +} + +show_summary() { + banner "Demo Complete!" + + echo "${white}What we demonstrated:${reset}" + echo "${green} ✓ Built separate images for API server and NVML provider${reset}" + echo "${green} ✓ Deployed as sidecar architecture via Helm${reset}" + echo "${green} ✓ Verified GPU registration through the sidecar${reset}" + echo "${green} ✓ Showed crash isolation and recovery${reset}" + echo "${green} ✓ Explored provider metrics${reset}" + echo "" + echo "${white}Key files:${reset}" + echo "${cyan} - Dockerfile.nvml-provider # Sidecar container build${reset}" + echo "${cyan} - values-sidecar-test.yaml # Helm values for sidecar mode${reset}" + echo "${cyan} - charts/device-api-server/ # Helm chart with sidecar support${reset}" + echo "" + echo "${white}Learn more:${reset}" + echo "${cyan} - docs/design/nvml-containerization-decision.md${reset}" + echo "${cyan} - docs/design/nvml-container-architecture-exploration.md${reset}" + echo "" +} + +# ============================================================================== +# Main +# ============================================================================== + +main() { + export KUBECONFIG + + show_intro + check_prereqs + show_cluster_info + + if confirm "Build and push container images?"; then + build_images + else + info "Skipping image build. Using existing images at ttl.sh" + fi + + show_values_file + + if confirm "Deploy the sidecar architecture to the cluster?"; then + deploy_sidecar + verify_gpu_registration + demonstrate_crash_recovery + show_metrics + cleanup + else + info "Skipping deployment" + fi + + show_summary +} + +# Run main if script is executed (not sourced) +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi diff --git a/deployments/container/Dockerfile b/deployments/container/Dockerfile new file mode 100644 index 000000000..472b27da7 --- /dev/null +++ b/deployments/container/Dockerfile @@ -0,0 +1,183 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Multi-target Dockerfile for NVSentinel components +# +# Targets: +# device-api-server - Pure Go server (no NVML, uses sidecar provider) +# nvml-provider - NVML provider sidecar (CGO, requires RuntimeClass nvidia) +# +# Build examples: +# # Build device-api-server (default, pure Go) +# docker build --target device-api-server -t nvsentinel/device-api-server . +# +# # Build nvml-provider sidecar +# docker build --target nvml-provider -t nvsentinel/nvml-provider . +# +# Note: NVML provider requires glibc runtime (Debian) for RTLD_DEEPBIND support + +# ============================================================================== +# Build Arguments +# ============================================================================== + +ARG GOLANG_VERSION=1.25 +ARG VERSION=dev +ARG GIT_COMMIT=unknown +ARG GIT_TREE_STATE=dirty +ARG BUILD_DATE + +# ============================================================================== +# Base Builder - Pure Go (Alpine) +# ============================================================================== + +FROM golang:${GOLANG_VERSION}-alpine AS builder-alpine + +ARG VERSION +ARG GIT_COMMIT +ARG GIT_TREE_STATE +ARG BUILD_DATE + +WORKDIR /workspace + +# Install build dependencies +RUN apk add --no-cache git make + +# Copy go mod files first for caching +COPY go.mod go.sum ./ +COPY api/go.mod api/go.sum ./api/ + +# Download dependencies +RUN go mod download + +# Copy source code +COPY . . + +# Version package path +ARG VERSION_PKG=github.com/nvidia/nvsentinel/pkg/version + +# Build device-api-server (CGO disabled, pure Go) +RUN CGO_ENABLED=0 GOOS=linux go build \ + -ldflags "-s -w \ + -X ${VERSION_PKG}.Version=${VERSION} \ + -X ${VERSION_PKG}.GitCommit=${GIT_COMMIT} \ + -X ${VERSION_PKG}.GitTreeState=${GIT_TREE_STATE} \ + -X ${VERSION_PKG}.BuildDate=${BUILD_DATE}" \ + -o /build/device-api-server \ + ./cmd/device-api-server + +# ============================================================================== +# Base Builder - CGO (Debian/glibc) +# ============================================================================== + +FROM golang:${GOLANG_VERSION}-bookworm AS builder-debian + +ARG VERSION +ARG GIT_COMMIT +ARG GIT_TREE_STATE +ARG BUILD_DATE + +WORKDIR /workspace + +# Install build dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + make \ + && rm -rf /var/lib/apt/lists/* + +# Copy go mod files first for caching +COPY go.mod go.sum ./ +COPY api/go.mod api/go.sum ./api/ + +# Download dependencies +RUN go mod download + +# Copy source code +COPY . . + +# Version package path +ARG VERSION_PKG=github.com/nvidia/nvsentinel/pkg/version + +# Build nvml-provider (CGO enabled for go-nvml) +RUN CGO_ENABLED=1 go build \ + -tags=nvml \ + -ldflags "-s -w \ + -X ${VERSION_PKG}.Version=${VERSION} \ + -X ${VERSION_PKG}.GitCommit=${GIT_COMMIT} \ + -X ${VERSION_PKG}.GitTreeState=${GIT_TREE_STATE} \ + -X ${VERSION_PKG}.BuildDate=${BUILD_DATE}" \ + -o /build/nvml-provider \ + ./cmd/nvml-provider + +# ============================================================================== +# Target: device-api-server +# ============================================================================== +# Pure Go server with no NVML dependencies. Uses sidecar provider for GPU access. +# Small image size, fast startup, works on any architecture. + +FROM alpine:3.21 AS device-api-server + +LABEL org.opencontainers.image.source="https://github.com/nvidia/nvsentinel" +LABEL org.opencontainers.image.description="NVSentinel Device API Server - Node-local GPU device state cache" +LABEL org.opencontainers.image.licenses="Apache-2.0" +LABEL org.opencontainers.image.title="device-api-server" + +# Add ca-certificates for HTTPS +RUN apk add --no-cache ca-certificates + +WORKDIR / + +COPY --from=builder-alpine --chmod=755 /build/device-api-server /device-api-server + +# Run as non-root user (nobody) +USER 65534:65534 + +# gRPC API port (default, configurable via --grpc-address) +EXPOSE 50051 +# Health check port (default, configurable via --health-port) +EXPOSE 8081 +# Metrics port (default, configurable via --metrics-port) +EXPOSE 9090 + +ENTRYPOINT ["/device-api-server"] + +# ============================================================================== +# Target: nvml-provider +# ============================================================================== +# NVML provider sidecar for GPU enumeration and health monitoring. +# Requires glibc runtime (Debian) for RTLD_DEEPBIND support. +# Must run with RuntimeClass: nvidia to access NVML libraries. + +FROM debian:bookworm-slim AS nvml-provider + +LABEL org.opencontainers.image.source="https://github.com/nvidia/nvsentinel" +LABEL org.opencontainers.image.description="NVSentinel NVML Provider - GPU enumeration and health monitoring sidecar" +LABEL org.opencontainers.image.licenses="Apache-2.0" +LABEL org.opencontainers.image.title="nvml-provider" + +# Add ca-certificates for HTTPS +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR / + +COPY --from=builder-debian --chmod=755 /build/nvml-provider /nvml-provider + +# Run as non-root user +USER 65534:65534 + +# Health check port +EXPOSE 8082 + +ENTRYPOINT ["/nvml-provider"] diff --git a/deployments/helm/health-provider/Chart.yaml b/deployments/helm/health-provider/Chart.yaml new file mode 100644 index 000000000..0ff7ffb1a --- /dev/null +++ b/deployments/helm/health-provider/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: health-provider +description: GPU Health Provider for NVSentinel - monitors GPU health via NVML + +type: application +version: 0.1.0 +appVersion: "0.1.0" + +keywords: + - nvidia + - gpu + - health + - monitoring + - nvml + +home: https://github.com/nvidia/nvsentinel +sources: + - https://github.com/nvidia/nvsentinel + +maintainers: + - name: NVIDIA + url: https://github.com/nvidia + +dependencies: [] diff --git a/deployments/helm/health-provider/templates/_helpers.tpl b/deployments/helm/health-provider/templates/_helpers.tpl new file mode 100644 index 000000000..b7890d71b --- /dev/null +++ b/deployments/helm/health-provider/templates/_helpers.tpl @@ -0,0 +1,61 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "health-provider.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "health-provider.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "health-provider.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "health-provider.labels" -}} +helm.sh/chart: {{ include "health-provider.chart" . }} +{{ include "health-provider.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "health-provider.selectorLabels" -}} +app.kubernetes.io/name: {{ include "health-provider.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: health-provider +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "health-provider.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "health-provider.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/deployments/helm/health-provider/templates/daemonset.yaml b/deployments/helm/health-provider/templates/daemonset.yaml new file mode 100644 index 000000000..51d5bddac --- /dev/null +++ b/deployments/helm/health-provider/templates/daemonset.yaml @@ -0,0 +1,98 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "health-provider.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "health-provider.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "health-provider.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "health-provider.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "health-provider.serviceAccountName" . }} + {{- if .Values.runtimeClassName }} + runtimeClassName: {{ .Values.runtimeClassName }} + {{- end }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --server-address={{ .Values.server.address }} + - --driver-root={{ .Values.nvml.driverRoot }} + - --enable-nvml={{ .Values.nvml.enabled }} + - --health-port={{ .Values.healthCheck.port }} + - -v={{ .Values.logVerbosity }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- with .Values.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: health + containerPort: {{ .Values.healthCheck.port }} + protocol: TCP + {{- with .Values.healthCheck.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.healthCheck.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deployments/helm/health-provider/templates/serviceaccount.yaml b/deployments/helm/health-provider/templates/serviceaccount.yaml new file mode 100644 index 000000000..013419c55 --- /dev/null +++ b/deployments/helm/health-provider/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "health-provider.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "health-provider.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/deployments/helm/health-provider/values.yaml b/deployments/helm/health-provider/values.yaml new file mode 100644 index 000000000..5102ad49f --- /dev/null +++ b/deployments/helm/health-provider/values.yaml @@ -0,0 +1,101 @@ +# Default values for health-provider +# This is a YAML-formatted file. + +# Number of replicas (DaemonSet uses nodeSelector instead) +replicaCount: 1 + +image: + repository: nvcr.io/nvidia/nvsentinel/health-provider + pullPolicy: IfNotPresent + tag: "" # Overrides the chart appVersion + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + annotations: {} + name: "" + +# Pod annotations +podAnnotations: {} + +# Pod security context +podSecurityContext: {} + +# Container security context - requires privileged for NVML +securityContext: + privileged: true + +# Server configuration +server: + # Address of device-api-server gRPC endpoint + address: "localhost:9001" + +# NVML configuration +nvml: + # Enable NVML GPU health monitoring + enabled: true + # Root path for NVIDIA driver libraries + driverRoot: "/run/nvidia/driver" + +# Health check configuration +healthCheck: + # HTTP port for liveness/readiness probes + port: 8082 + livenessProbe: + httpGet: + path: /livez + port: health + initialDelaySeconds: 10 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /readyz + port: health + initialDelaySeconds: 5 + periodSeconds: 5 + +# Resource limits and requests +resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 10m + memory: 32Mi + +# Node selector - use to restrict to GPU nodes +nodeSelector: {} + # nvidia.com/gpu.present: "true" + +# Tolerations - tolerate GPU-related taints +tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + +# Affinity rules +affinity: {} + +# RuntimeClassName for NVIDIA GPU access without consuming GPUs +runtimeClassName: nvidia + +# Volume mounts for NVIDIA driver libraries +volumes: + - name: driver-root + hostPath: + path: /run/nvidia/driver + type: DirectoryOrCreate + +volumeMounts: + - name: driver-root + mountPath: /run/nvidia/driver + readOnly: true + +# Environment variables +env: [] + +# Logging verbosity (klog -v flag) +logVerbosity: 2 diff --git a/deployments/helm/nvsentinel/Chart.yaml b/deployments/helm/nvsentinel/Chart.yaml new file mode 100644 index 000000000..0cfdf39a3 --- /dev/null +++ b/deployments/helm/nvsentinel/Chart.yaml @@ -0,0 +1,51 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v2 +name: nvsentinel +description: | + NVSentinel - Node-local GPU device state cache server for Kubernetes. + + The Device API Server acts as an intermediary between providers (health monitors) + that update GPU device states and consumers (device plugins, DRA drivers) that + read device states for scheduling decisions. + + Key features: + - Read-blocking semantics during provider updates + - Sidecar architecture for NVML isolation + - Multiple provider and consumer support + - Prometheus metrics and alerting + - Health-based GPU scheduling decisions +type: application +version: 0.1.0 +appVersion: "0.1.0" +kubeVersion: ">=1.25.0-0" +keywords: + - nvidia + - gpu + - device + - nvml + - health + - daemonset + - grpc +home: https://github.com/nvidia/nvsentinel +sources: + - https://github.com/nvidia/nvsentinel +maintainers: + - name: NVIDIA + url: https://github.com/nvidia +icon: https://www.nvidia.com/favicon.ico +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/category: monitoring-logging diff --git a/deployments/helm/nvsentinel/README.md b/deployments/helm/nvsentinel/README.md new file mode 100644 index 000000000..e4bc9598e --- /dev/null +++ b/deployments/helm/nvsentinel/README.md @@ -0,0 +1,264 @@ +# Device API Server Helm Chart + +Node-local GPU device state cache server for Kubernetes. + +## Introduction + +The Device API Server is a DaemonSet that runs on each GPU node, providing a local gRPC cache for GPU device states. It acts as an intermediary between: + +- **Providers** (health monitors) that update GPU device states +- **Consumers** (device plugins, DRA drivers) that read device states for scheduling decisions + +Key features: + +- Read-blocking semantics during provider updates +- Multiple provider and consumer support +- Optional NVML fallback provider for GPU enumeration and XID monitoring +- Prometheus metrics and alerting +- Unix socket for node-local communication + +## Prerequisites + +- Kubernetes 1.25+ +- Helm 3.0+ +- (Optional) NVIDIA GPU Operator for NVML provider support +- (Optional) Prometheus Operator for ServiceMonitor/PrometheusRule + +## Installation + +### Quick Start + +```bash +# Add the Helm repository (when published) +helm repo add device-api https://nvidia.github.io/device-api +helm repo update + +# Install with default configuration +helm install device-api-server device-api/device-api-server \ + --namespace device-api --create-namespace +``` + +### Install from Local Chart + +```bash +helm install device-api-server ./charts/device-api-server \ + --namespace device-api --create-namespace +``` + +### Install with NVML Provider + +To enable built-in GPU enumeration and health monitoring via NVML: + +```bash +helm install device-api-server ./charts/device-api-server \ + --namespace device-api --create-namespace \ + --set nvml.enabled=true +``` + +> **Note**: NVML provider requires the `nvidia` RuntimeClass. Install the NVIDIA GPU Operator or create it manually. + +### Install with Prometheus Monitoring + +```bash +helm install device-api-server ./charts/device-api-server \ + --namespace device-api --create-namespace \ + --set metrics.serviceMonitor.enabled=true \ + --set metrics.prometheusRule.enabled=true +``` + +## Configuration + +See [values.yaml](values.yaml) for the full list of configurable parameters. + +### Key Parameters + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `image.repository` | Image repository | `ghcr.io/nvidia/device-api-server` | +| `image.tag` | Image tag | Chart appVersion | +| `server.grpcAddress` | gRPC server address | `:50051` | +| `server.unixSocket` | Unix socket path | `/var/run/device-api/device.sock` | +| `server.healthPort` | Health endpoint port | `8081` | +| `server.metricsPort` | Metrics endpoint port | `9090` | +| `nvml.enabled` | Enable NVML fallback provider | `false` | +| `nvml.driverRoot` | NVIDIA driver library root | `/run/nvidia/driver` | +| `nvml.healthCheckEnabled` | Enable XID event monitoring | `true` | +| `runtimeClassName` | Pod RuntimeClass | `""` (auto `nvidia` if nvml.enabled) | +| `nodeSelector` | Node selector | `nvidia.com/gpu.present: "true"` | +| `metrics.serviceMonitor.enabled` | Create ServiceMonitor | `false` | +| `metrics.prometheusRule.enabled` | Create PrometheusRule | `false` | + +### Resource Configuration + +```yaml +resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 256Mi +``` + +### NVML Provider Configuration + +```yaml +nvml: + enabled: true + driverRoot: /run/nvidia/driver + additionalIgnoredXids: "" # Comma-separated list of XIDs to ignore + healthCheckEnabled: true +``` + +Default ignored XIDs (application errors): 13, 31, 43, 45, 68, 109 + +### Node Scheduling + +By default, the DaemonSet schedules only on nodes with `nvidia.com/gpu.present=true` label: + +```yaml +nodeSelector: + nvidia.com/gpu.present: "true" + +tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule +``` + +Override for custom environments: + +```bash +helm install device-api-server ./charts/device-api-server \ + --set 'nodeSelector.node-type=gpu' \ + --set 'nodeSelector.nvidia\.com/gpu\.present=null' +``` + +## Metrics + +The server exposes Prometheus metrics at `/metrics` on the configured `metricsPort`. + +### Available Metrics + +| Metric | Type | Description | +|--------|------|-------------| +| `device_api_server_info` | Gauge | Server information | +| `device_api_server_cache_gpus_total` | Gauge | Total GPUs in cache | +| `device_api_server_cache_gpus_healthy` | Gauge | Healthy GPUs | +| `device_api_server_cache_gpus_unhealthy` | Gauge | Unhealthy GPUs | +| `device_api_server_cache_updates_total` | Counter | Cache update operations | +| `device_api_server_watch_streams_active` | Gauge | Active watch streams | +| `device_api_server_watch_events_total` | Counter | Watch events sent | +| `device_api_server_nvml_provider_enabled` | Gauge | NVML provider status | +| `device_api_server_nvml_gpu_count` | Gauge | GPUs discovered by NVML | + +### Alerting Rules + +When `metrics.prometheusRule.enabled=true`, the following alerts are configured: + +| Alert | Severity | Description | +|-------|----------|-------------| +| `DeviceAPIServerDown` | Critical | Server unreachable for 5m | +| `DeviceAPIServerHighLatency` | Warning | P99 latency > 500ms | +| `DeviceAPIServerHighErrorRate` | Warning | Error rate > 10% | +| `DeviceAPIServerUnhealthyGPUs` | Warning | Unhealthy GPUs detected | +| `DeviceAPIServerNoGPUs` | Warning | No GPUs registered for 10m | +| `DeviceAPIServerNVMLProviderDown` | Warning | NVML provider not running | + +## Client Connection + +Clients on the same node can connect via: + +### Unix Socket (Recommended) + +```go +conn, err := grpc.Dial( + "unix:///var/run/device-api/device.sock", + grpc.WithInsecure(), +) +``` + +### TCP + +```go +conn, err := grpc.Dial( + "localhost:50051", + grpc.WithInsecure(), +) +``` + +### grpcurl Examples + +```bash +# List available services +grpcurl -plaintext localhost:50051 list + +# List GPUs +grpcurl -plaintext localhost:50051 nvidia.device.v1alpha1.GpuService/ListGpus + +# Watch GPU changes +grpcurl -plaintext localhost:50051 nvidia.device.v1alpha1.GpuService/WatchGpus +``` + +## Upgrading + +```bash +helm upgrade device-api-server ./charts/device-api-server \ + --namespace device-api \ + --reuse-values \ + --set image.tag=v0.2.0 +``` + +## Uninstallation + +```bash +helm uninstall device-api-server --namespace device-api +``` + +## Troubleshooting + +### Pod Not Scheduling + +Check node labels: + +```bash +kubectl get nodes --show-labels | grep gpu +``` + +Ensure nodes have `nvidia.com/gpu.present=true` or override `nodeSelector`. + +### NVML Provider Fails to Start + +1. Verify RuntimeClass exists: + + ```bash + kubectl get runtimeclass nvidia + ``` + +2. Check NVIDIA driver is installed on nodes: + + ```bash + kubectl debug node/ -it --image=nvidia/cuda:12.0-base -- nvidia-smi + ``` + +3. Check pod logs for NVML errors: + + ```bash + kubectl logs -n device-api -l app.kubernetes.io/name=device-api-server + ``` + +### Permission Denied on Unix Socket + +If using custom security contexts, ensure the socket directory is writable: + +```yaml +securityContext: + runAsUser: 0 # May be needed for hostPath access + runAsNonRoot: false +``` + +## License + +Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + +Licensed under the Apache License, Version 2.0. diff --git a/deployments/helm/nvsentinel/crds/nvsentinel.nvidia.com_healthevents.yaml b/deployments/helm/nvsentinel/crds/nvsentinel.nvidia.com_healthevents.yaml new file mode 100644 index 000000000..10768d603 --- /dev/null +++ b/deployments/helm/nvsentinel/crds/nvsentinel.nvidia.com_healthevents.yaml @@ -0,0 +1,361 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: healthevents.nvsentinel.nvidia.com +spec: + group: nvsentinel.nvidia.com + names: + kind: HealthEvent + listKind: HealthEventList + plural: healthevents + shortNames: + - he + - hevt + singular: healthevent + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.nodeName + name: Node + type: string + - jsonPath: .spec.source + name: Source + type: string + - jsonPath: .spec.isFatal + name: Fatal + type: boolean + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + HealthEvent represents a GPU health event detected by a health monitor. + It replaces the MongoDB HealthEventWithStatus document and enables + Kubernetes-native coordination between fault-quarantine, node-drainer, + and remediation controllers. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + HealthEventSpec defines the immutable details of a health event. + These fields are set when the event is created and should not be modified. + properties: + agent: + description: Agent is the name of the agent that reported the event + (for compatibility with protobuf). + type: string + checkName: + description: |- + CheckName identifies the specific health check that detected the issue. + Examples: "xid-error-check", "memory-ecc-check", "thermal-check" + type: string + componentClass: + description: |- + ComponentClass identifies the type of component affected. + Examples: "GPU", "NIC", "Storage", "Memory" + type: string + detectedAt: + description: DetectedAt is the timestamp when the event was first + detected. + format: date-time + type: string + entitiesImpacted: + description: EntitiesImpacted lists the specific entities affected + by this event. + items: + description: Entity represents an impacted entity (GPU, NIC, etc.). + properties: + type: + description: Type identifies the kind of entity (e.g., "GPU", + "NIC"). + type: string + value: + description: Value is the identifier for the entity (e.g., GPU + UUID). + type: string + required: + - type + - value + type: object + type: array + errorCodes: + description: |- + ErrorCodes contains the error codes associated with this event. + For GPU events, these are typically XID error codes (e.g., "79", "31"). + items: + type: string + type: array + isFatal: + default: false + description: |- + IsFatal indicates whether this event requires immediate node quarantine. + Fatal events trigger the full fault-handling workflow. + type: boolean + isHealthy: + default: false + description: |- + IsHealthy indicates the current health state. + false = unhealthy (problem detected), true = healthy (recovered) + type: boolean + maintenance: + description: |- + Maintenance contains CSP maintenance event details. + Only populated when Source is "csp-health-monitor". + properties: + actualEndTime: + description: ActualEndTime is when maintenance actually ended. + format: date-time + type: string + actualStartTime: + description: ActualStartTime is when maintenance actually started. + format: date-time + type: string + csp: + description: CSP identifies the cloud service provider. + enum: + - aws + - gcp + type: string + cspStatus: + description: CSPStatus is the status reported by the CSP. + type: string + eventId: + description: EventID is the CSP's identifier for this maintenance + event. + type: string + maintenanceType: + description: MaintenanceType indicates if this is scheduled or + unscheduled. + enum: + - SCHEDULED + - UNSCHEDULED + type: string + scheduledEndTime: + description: ScheduledEndTime is when maintenance is scheduled + to end. + format: date-time + type: string + scheduledStartTime: + description: ScheduledStartTime is when maintenance is scheduled + to begin. + format: date-time + type: string + required: + - csp + - eventId + - maintenanceType + type: object + message: + description: Message provides a human-readable description of the + event. + type: string + metadata: + additionalProperties: + type: string + description: |- + Metadata contains additional key-value data about the event. + Common keys: "uuid", "temperature", "serialNumber" + type: object + nodeName: + description: NodeName is the Kubernetes node where the event occurred. + type: string + overrides: + description: Overrides allows customizing the fault-handling behavior + for this event. + properties: + drain: + description: Drain overrides control pod eviction behavior. + properties: + force: + default: false + description: Force causes the action to proceed even if conditions + aren't met. + type: boolean + skip: + default: false + description: Skip causes the action to be skipped entirely. + type: boolean + type: object + quarantine: + description: Quarantine overrides control node cordoning behavior. + properties: + force: + default: false + description: Force causes the action to proceed even if conditions + aren't met. + type: boolean + skip: + default: false + description: Skip causes the action to be skipped entirely. + type: boolean + type: object + type: object + recommendedAction: + allOf: + - enum: + - NONE + - COMPONENT_RESET + - CONTACT_SUPPORT + - RUN_FIELDDIAG + - RESTART_VM + - RESTART_BM + - REPLACE_VM + - RUN_DCGMEUD + - UNKNOWN + - enum: + - NONE + - COMPONENT_RESET + - CONTACT_SUPPORT + - RUN_FIELDDIAG + - RESTART_VM + - RESTART_BM + - REPLACE_VM + - RUN_DCGMEUD + - UNKNOWN + default: NONE + description: RecommendedAction suggests what action should be taken. + type: string + source: + description: |- + Source identifies the health monitor that detected this event. + Examples: "nvml-health-monitor", "dcgm", "csp-health-monitor", "syslog-monitor" + type: string + required: + - checkName + - componentClass + - detectedAt + - isFatal + - isHealthy + - nodeName + - source + type: object + status: + description: |- + HealthEventStatus defines the current processing state of a health event. + These fields are updated by controllers as the event moves through the workflow. + properties: + conditions: + description: Conditions provide detailed status for each stage of + processing. + items: + description: |- + HealthEventCondition represents a condition of a HealthEvent. + Follows the standard Kubernetes condition pattern. + properties: + lastTransitionTime: + description: LastTransitionTime is the last time the condition + transitioned. + format: date-time + type: string + message: + description: Message is a human-readable description of the + condition. + type: string + observedGeneration: + description: ObservedGeneration is the generation of the resource + when this condition was set. + format: int64 + type: integer + reason: + description: Reason is a brief machine-readable reason for the + condition. + type: string + status: + description: Status is the status of the condition (True, False, + Unknown). + enum: + - "True" + - "False" + - Unknown + type: string + type: + allOf: + - enum: + - Detected + - NodeQuarantined + - PodsDrained + - Remediated + - Resolved + - enum: + - Detected + - NodeQuarantined + - PodsDrained + - Remediated + - Resolved + description: Type is the type of condition. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + lastRemediationTime: + description: LastRemediationTime is when remediation was last attempted. + format: date-time + type: string + observedGeneration: + description: ObservedGeneration is the generation observed by the + controller. + format: int64 + type: integer + phase: + allOf: + - enum: + - New + - Quarantined + - Draining + - Drained + - Remediating + - Remediated + - Resolved + - Cancelled + - enum: + - New + - Quarantined + - Draining + - Drained + - Remediating + - Remediated + - Resolved + - Cancelled + default: New + description: Phase represents the current state in the fault-handling + workflow. + type: string + resolvedAt: + description: |- + ResolvedAt is when the event reached the Resolved phase. + Used for TTL calculation. + format: date-time + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deployments/helm/nvsentinel/templates/NOTES.txt b/deployments/helm/nvsentinel/templates/NOTES.txt new file mode 100644 index 000000000..2e2fb988a --- /dev/null +++ b/deployments/helm/nvsentinel/templates/NOTES.txt @@ -0,0 +1,127 @@ +{{/* +Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +=============================================================================== + NVIDIA Device API Server has been installed! +=============================================================================== + +Release: {{ .Release.Name }} +Namespace: {{ .Release.Namespace }} +Chart Version: {{ .Chart.Version }} +App Version: {{ .Chart.AppVersion }} + +------------------------------------------------------------------------------- + Configuration Summary +------------------------------------------------------------------------------- + +gRPC Address: {{ .Values.server.grpcAddress }} +Unix Socket: {{ .Values.server.unixSocket }} +Health Port: {{ .Values.server.healthPort }} +Metrics Port: {{ .Values.server.metricsPort }} +{{- if .Values.nvml.enabled }} +NVML Provider: Enabled + - Driver Root: {{ .Values.nvml.driverRoot }} + - Health Check: {{ .Values.nvml.healthCheckEnabled }} +{{- else }} +NVML Provider: Disabled +{{- end }} + +------------------------------------------------------------------------------- + Verify Installation +------------------------------------------------------------------------------- + +1. Check that DaemonSet pods are running on GPU nodes: + + kubectl get pods -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} -o wide + +2. Check pod logs: + + kubectl logs -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} -f + +3. Verify health endpoint (from within the cluster): + + kubectl run -n {{ .Release.Namespace }} --rm -it --restart=Never --image=curlimages/curl:latest curl -- \ + curl -s http://{{ include "device-api-server.fullname" . }}-metrics.{{ .Release.Namespace }}.svc:{{ .Values.server.metricsPort }}/metrics | head -20 + +{{- if .Values.metrics.enabled }} + +------------------------------------------------------------------------------- + Metrics & Monitoring +------------------------------------------------------------------------------- + +Metrics endpoint: http://:{{ .Values.server.metricsPort }}/metrics + +{{- if .Values.metrics.serviceMonitor.enabled }} +ServiceMonitor: Enabled (Prometheus will auto-discover) +{{- else }} +ServiceMonitor: Disabled + To enable Prometheus auto-discovery, upgrade with: + --set metrics.serviceMonitor.enabled=true +{{- end }} + +{{- if .Values.metrics.prometheusRule.enabled }} +PrometheusRule: Enabled (alerts configured) +{{- else }} +PrometheusRule: Disabled + To enable alerting rules, upgrade with: + --set metrics.prometheusRule.enabled=true +{{- end }} +{{- end }} + +------------------------------------------------------------------------------- + Client Connection +------------------------------------------------------------------------------- + +Providers and consumers on the same node can connect via: + + - Unix Socket: {{ .Values.server.unixSocket }} + - gRPC (TCP): localhost{{ .Values.server.grpcAddress }} + +Example using grpcurl: + + # List available services + grpcurl -plaintext localhost:{{ .Values.server.grpcAddress | trimPrefix ":" }} list + + # List GPUs + grpcurl -plaintext localhost:{{ .Values.server.grpcAddress | trimPrefix ":" }} \ + nvidia.device.v1alpha1.GpuService/ListGpus + +{{- if .Values.nvml.enabled }} + +------------------------------------------------------------------------------- + NVML Provider Notes +------------------------------------------------------------------------------- + +The NVML provider requires: + 1. RuntimeClass "nvidia" must exist in the cluster + 2. NVIDIA GPU Operator or Container Toolkit installed + 3. Nodes must have NVIDIA GPUs + +Verify RuntimeClass exists: + kubectl get runtimeclass nvidia + +If not present, create it or install the NVIDIA GPU Operator: + https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/ + +{{- end }} + +------------------------------------------------------------------------------- + Support +------------------------------------------------------------------------------- + +Documentation: https://github.com/nvidia/device-api +Issues: https://github.com/nvidia/device-api/issues + +=============================================================================== diff --git a/deployments/helm/nvsentinel/templates/_helpers.tpl b/deployments/helm/nvsentinel/templates/_helpers.tpl new file mode 100644 index 000000000..b55c80319 --- /dev/null +++ b/deployments/helm/nvsentinel/templates/_helpers.tpl @@ -0,0 +1,95 @@ +{{/* +Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +Expand the name of the chart. +*/}} +{{- define "device-api-server.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "device-api-server.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "device-api-server.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "device-api-server.labels" -}} +helm.sh/chart: {{ include "device-api-server.chart" . }} +{{ include "device-api-server.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/part-of: device-api +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "device-api-server.selectorLabels" -}} +app.kubernetes.io/name: {{ include "device-api-server.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: device-api-server +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "device-api-server.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "device-api-server.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Create the image name +*/}} +{{- define "device-api-server.image" -}} +{{- $tag := default .Chart.AppVersion .Values.image.tag -}} +{{- printf "%s:%s" .Values.image.repository $tag }} +{{- end }} + +{{/* +Socket directory path +*/}} +{{- define "device-api-server.socketDir" -}} +{{- .Values.server.unixSocket | dir }} +{{- end }} diff --git a/deployments/helm/nvsentinel/templates/daemonset.yaml b/deployments/helm/nvsentinel/templates/daemonset.yaml new file mode 100644 index 000000000..6ee2640fa --- /dev/null +++ b/deployments/helm/nvsentinel/templates/daemonset.yaml @@ -0,0 +1,220 @@ +{{/* +Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "device-api-server.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "device-api-server.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "device-api-server.selectorLabels" . | nindent 6 }} + updateStrategy: + {{- toYaml .Values.updateStrategy | nindent 4 }} + template: + metadata: + annotations: + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "device-api-server.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "device-api-server.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- if or .Values.runtimeClassName (and .Values.nvml.enabled (not .Values.runtimeClassName)) }} + runtimeClassName: {{ .Values.runtimeClassName | default "nvidia" }} + {{- end }} + {{- with .Values.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: {{ include "device-api-server.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --grpc-address={{ .Values.server.grpcAddress }} + - --unix-socket={{ .Values.server.unixSocket }} + - --health-port={{ .Values.server.healthPort }} + - --metrics-port={{ .Values.server.metricsPort }} + - --shutdown-timeout={{ .Values.server.shutdownTimeout }} + - --shutdown-delay={{ .Values.server.shutdownDelay }} + - --log-format={{ .Values.logging.format }} + - -v={{ .Values.logging.verbosity }} + {{- /* nvmlProvider.enabled doesn't require args on server - the sidecar connects to the server */}} + {{- if .Values.nvml.enabled }} + - --enable-nvml-provider=true + - --nvml-driver-root={{ .Values.nvml.driverRoot }} + - --nvml-health-check={{ .Values.nvml.healthCheckEnabled }} + {{- if .Values.nvml.additionalIgnoredXids }} + - --nvml-ignored-xids={{ .Values.nvml.additionalIgnoredXids }} + {{- end }} + {{- end }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if .Values.nvml.enabled }} + # NVIDIA Container Toolkit environment variables + # These configure NVML access without consuming GPU resources + - name: NVIDIA_VISIBLE_DEVICES + value: "all" + - name: NVIDIA_DRIVER_CAPABILITIES + value: "utility" + {{- end }} + {{- with .Values.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + lifecycle: + preStop: + exec: + # Sleep to allow k8s to propagate endpoint removal + command: ["sleep", "{{ .Values.server.shutdownDelay }}"] + ports: + - name: grpc + containerPort: {{ .Values.server.grpcAddress | trimPrefix ":" | int }} + protocol: TCP + - name: health + containerPort: {{ .Values.server.healthPort }} + protocol: TCP + {{- if .Values.metrics.enabled }} + - name: metrics + containerPort: {{ .Values.server.metricsPort }} + protocol: TCP + {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: socket-dir + mountPath: {{ include "device-api-server.socketDir" . }} + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.nvmlProvider.enabled }} + # NVML Provider sidecar container + - name: nvml-provider + image: "{{ .Values.nvmlProvider.image.repository }}:{{ .Values.nvmlProvider.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.nvmlProvider.image.pullPolicy }} + args: + - --server-address={{ .Values.nvmlProvider.serverAddress }} + - --provider-id={{ .Values.nvmlProvider.providerID }} + - --driver-root={{ .Values.nvmlProvider.driverRoot }} + - --health-check={{ .Values.nvmlProvider.healthCheckEnabled }} + - --health-port={{ .Values.nvmlProvider.healthPort }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # NVIDIA Container Toolkit environment variables + - name: NVIDIA_VISIBLE_DEVICES + value: "all" + - name: NVIDIA_DRIVER_CAPABILITIES + value: "utility" + ports: + - name: provider-health + containerPort: {{ .Values.nvmlProvider.healthPort }} + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: provider-health + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: provider-health + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 3 + {{- with .Values.nvmlProvider.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.nvmlProvider.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- with .Values.sidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: socket-dir + hostPath: + path: {{ include "device-api-server.socketDir" . }} + type: DirectoryOrCreate + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + # terminationGracePeriodSeconds = preStop sleep + shutdown delay + shutdown timeout + buffer + terminationGracePeriodSeconds: {{ add .Values.server.shutdownDelay .Values.server.shutdownDelay .Values.server.shutdownTimeout 5 }} diff --git a/deployments/helm/nvsentinel/templates/policy-configmap.yaml b/deployments/helm/nvsentinel/templates/policy-configmap.yaml new file mode 100644 index 000000000..e02145d57 --- /dev/null +++ b/deployments/helm/nvsentinel/templates/policy-configmap.yaml @@ -0,0 +1,72 @@ +{{/* +Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- if .Values.healthEvents.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "device-api-server.fullname" . }}-policy + namespace: {{ .Release.Namespace }} + labels: + {{- include "device-api-server.labels" . | nindent 4 }} + app.kubernetes.io/component: policy +data: + # ========================================================================== + # HealthEvent TTL Policy + # ========================================================================== + # Controls automatic cleanup of resolved HealthEvent CRs. + # The TTL controller deletes events after they remain in "Resolved" phase + # for the specified duration. + + # Time-to-live after event reaches Resolved phase. + # Format: Go duration string (e.g., "168h" = 7 days, "720h" = 30 days) + # Set to "0" to disable automatic cleanup. + ttlAfterResolved: {{ .Values.healthEvents.ttl.afterResolved | quote }} + + # ========================================================================== + # Retention Policy + # ========================================================================== + # Overrides TTL for compliance requirements. + + # Retention mode: "standard" (uses ttlAfterResolved) or "compliance" (uses complianceRetention) + retentionMode: {{ .Values.healthEvents.ttl.retentionMode | quote }} + + # Retention period for compliance mode (e.g., "2160h" = 90 days) + # Only used when retentionMode is "compliance" + complianceRetention: {{ .Values.healthEvents.ttl.complianceRetention | quote }} + + # ========================================================================== + # Cleanup Behavior + # ========================================================================== + + # Maximum number of events to delete per reconciliation cycle. + # Prevents API server overload during bulk cleanup. + batchSize: {{ .Values.healthEvents.ttl.batchSize | quote }} + + # Interval between cleanup cycles. + # Format: Go duration string (e.g., "5m", "1h") + cleanupInterval: {{ .Values.healthEvents.ttl.cleanupInterval | quote }} + + # ========================================================================== + # Event History Limits + # ========================================================================== + # Optional limits to prevent unbounded growth. + + # Maximum events to retain per node (0 = unlimited) + maxEventsPerNode: {{ .Values.healthEvents.limits.maxEventsPerNode | quote }} + + # Maximum total events cluster-wide (0 = unlimited) + maxTotalEvents: {{ .Values.healthEvents.limits.maxTotalEvents | quote }} +{{- end }} diff --git a/deployments/helm/nvsentinel/templates/prometheusrule.yaml b/deployments/helm/nvsentinel/templates/prometheusrule.yaml new file mode 100644 index 000000000..8413ede09 --- /dev/null +++ b/deployments/helm/nvsentinel/templates/prometheusrule.yaml @@ -0,0 +1,140 @@ +{{/* +Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- if and .Values.metrics.enabled .Values.metrics.prometheusRule.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "device-api-server.fullname" . }} + namespace: {{ .Values.metrics.prometheusRule.namespace | default .Release.Namespace }} + labels: + {{- include "device-api-server.labels" . | nindent 4 }} + {{- with .Values.metrics.prometheusRule.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + groups: + - name: device-api-server + rules: + # Server availability + - alert: DeviceAPIServerDown + expr: up{job="{{ include "device-api-server.fullname" . }}-metrics"} == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Device API Server is down on {{ "{{ $labels.instance }}" }}" + description: "Device API Server has been unreachable for more than 5 minutes." + runbook_url: "https://github.com/nvidia/device-api/blob/main/docs/operations/device-api-server.md#alert-deviceapiserverdown" + + # High latency + - alert: DeviceAPIServerHighLatency + expr: | + histogram_quantile(0.99, + sum(rate(grpc_server_handling_seconds_bucket{ + grpc_service="nvidia.device.v1alpha1.GpuService" + }[5m])) by (le, instance) + ) > 0.5 + for: 5m + labels: + severity: warning + annotations: + summary: "Device API Server high latency on {{ "{{ $labels.instance }}" }}" + description: "P99 latency is above 500ms for more than 5 minutes." + runbook_url: "https://github.com/nvidia/device-api/blob/main/docs/operations/device-api-server.md#alert-deviceapiserverhighlatency" + + # High error rate + - alert: DeviceAPIServerHighErrorRate + expr: | + sum(rate(grpc_server_handled_total{ + grpc_code!="OK", + grpc_service=~"nvidia.device.v1alpha1.*" + }[5m])) by (instance) + / + sum(rate(grpc_server_handled_total{ + grpc_service=~"nvidia.device.v1alpha1.*" + }[5m])) by (instance) + > 0.1 + for: 5m + labels: + severity: warning + annotations: + summary: "Device API Server high error rate on {{ "{{ $labels.instance }}" }}" + description: "Error rate is above 10% for more than 5 minutes." + runbook_url: "https://github.com/nvidia/device-api/blob/main/docs/operations/device-api-server.md#alert-deviceapiserverhigherrorrate" + + # Unhealthy GPUs detected + - alert: DeviceAPIServerUnhealthyGPUs + expr: device_api_server_cache_gpus_unhealthy > 0 + for: 1m + labels: + severity: warning + annotations: + summary: "Unhealthy GPUs detected on {{ "{{ $labels.instance }}" }}" + description: "{{ "{{ $value }}" }} GPU(s) are in unhealthy state." + runbook_url: "https://github.com/nvidia/device-api/blob/main/docs/operations/device-api-server.md#alert-deviceapiserverunhealthygpus" + + # No GPUs registered + - alert: DeviceAPIServerNoGPUs + expr: device_api_server_cache_gpus_total == 0 + for: 10m + labels: + severity: warning + annotations: + summary: "No GPUs registered on {{ "{{ $labels.instance }}" }}" + description: "Device API Server has no GPUs registered for more than 10 minutes." + runbook_url: "https://github.com/nvidia/device-api/blob/main/docs/operations/device-api-server.md#alert-deviceapiservernogpus" + + # NVML provider not running (when expected) + - alert: DeviceAPIServerNVMLProviderDown + expr: device_api_server_nvml_provider_enabled == 0 + for: 5m + labels: + severity: warning + annotations: + summary: "NVML provider is not running on {{ "{{ $labels.instance }}" }}" + description: "The NVML provider is not enabled or failed to initialize. GPUs may not be monitored." + runbook_url: "https://github.com/nvidia/device-api/blob/main/docs/operations/device-api-server.md#alert-deviceapiservernvmlproviderdown" + + # NVML health monitor not running + - alert: DeviceAPIServerNVMLHealthMonitorDown + expr: | + device_api_server_nvml_provider_enabled == 1 + and device_api_server_nvml_health_monitor_running == 0 + and device_api_server_nvml_gpu_count > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "NVML health monitor is not running on {{ "{{ $labels.instance }}" }}" + description: "NVML provider is enabled but health monitoring is not running. XID errors may not be detected." + runbook_url: "https://github.com/nvidia/device-api/blob/main/docs/operations/device-api-server.md#alert-deviceapiservernvmlhealthmonitordown" + + # High memory usage + - alert: DeviceAPIServerHighMemory + expr: | + process_resident_memory_bytes{job="{{ include "device-api-server.fullname" . }}-metrics"} > 512 * 1024 * 1024 + for: 10m + labels: + severity: warning + annotations: + summary: "Device API Server high memory usage on {{ "{{ $labels.instance }}" }}" + description: "Memory usage is above 512MB for more than 10 minutes." + runbook_url: "https://github.com/nvidia/device-api/blob/main/docs/operations/device-api-server.md#alert-deviceapiserverhighmemory" + + {{- with .Values.metrics.prometheusRule.additionalRules }} + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/deployments/helm/nvsentinel/templates/service.yaml b/deployments/helm/nvsentinel/templates/service.yaml new file mode 100644 index 000000000..7288d2fdb --- /dev/null +++ b/deployments/helm/nvsentinel/templates/service.yaml @@ -0,0 +1,37 @@ +{{/* +Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- if .Values.metrics.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "device-api-server.fullname" . }}-metrics + namespace: {{ .Release.Namespace }} + labels: + {{- include "device-api-server.labels" . | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.server.metricsPort }} + targetPort: metrics + protocol: TCP + name: metrics + selector: + {{- include "device-api-server.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/deployments/helm/nvsentinel/templates/serviceaccount.yaml b/deployments/helm/nvsentinel/templates/serviceaccount.yaml new file mode 100644 index 000000000..6f30202ce --- /dev/null +++ b/deployments/helm/nvsentinel/templates/serviceaccount.yaml @@ -0,0 +1,29 @@ +{{/* +Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "device-api-server.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "device-api-server.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} +{{- end }} diff --git a/deployments/helm/nvsentinel/templates/servicemonitor.yaml b/deployments/helm/nvsentinel/templates/servicemonitor.yaml new file mode 100644 index 000000000..2bc9e465b --- /dev/null +++ b/deployments/helm/nvsentinel/templates/servicemonitor.yaml @@ -0,0 +1,47 @@ +{{/* +Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "device-api-server.fullname" . }} + namespace: {{ .Values.metrics.serviceMonitor.namespace | default .Release.Namespace }} + labels: + {{- include "device-api-server.labels" . | nindent 4 }} + {{- with .Values.metrics.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "device-api-server.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + endpoints: + - port: metrics + interval: {{ .Values.metrics.serviceMonitor.interval }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} + path: /metrics + {{- with .Values.metrics.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.metrics.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/deployments/helm/nvsentinel/values.yaml b/deployments/helm/nvsentinel/values.yaml new file mode 100644 index 000000000..f960087bd --- /dev/null +++ b/deployments/helm/nvsentinel/values.yaml @@ -0,0 +1,307 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for nvsentinel. +# This is a YAML-formatted file. + +# -- Number of replicas (ignored for DaemonSet, kept for consistency) +replicaCount: 1 + +# -- Image configuration +image: + # -- Image repository + repository: ghcr.io/nvidia/nvsentinel/device-api-server + # -- Image pull policy + pullPolicy: IfNotPresent + # -- Image tag (defaults to Chart appVersion) + tag: "" + +# -- Image pull secrets +imagePullSecrets: [] + +# -- Override the name of the chart +nameOverride: "" + +# -- Override the full name of the chart +fullnameOverride: "" + +# -- Server configuration +server: + # -- gRPC server TCP address + grpcAddress: ":50051" + # -- Unix socket path for node-local communication + unixSocket: /var/run/device-api/device.sock + # -- HTTP port for health endpoints (/healthz, /readyz) + healthPort: 8081 + # -- HTTP port for Prometheus metrics + metricsPort: 9090 + # -- Graceful shutdown timeout in seconds + shutdownTimeout: 30 + # -- Shutdown delay in seconds (allows k8s endpoint propagation) + shutdownDelay: 5 + +# -- Logging configuration +logging: + # -- Log verbosity level (0=info, higher=more verbose) + verbosity: 0 + # -- Log format: text or json + format: json + +# -- NVML Fallback Provider configuration (built-in) +# Enables built-in GPU enumeration and health monitoring via NVML. +# Uses the 'nvidia' RuntimeClass to inject NVML libraries without consuming GPU resources. +# NOTE: For containerized deployment, prefer nvmlProvider (sidecar) instead. +nvml: + # -- Enable the NVML fallback provider + enabled: false + # -- Root path where NVIDIA driver libraries are located + # Common values: /run/nvidia/driver (container), / (bare metal) + driverRoot: /run/nvidia/driver + # -- Comma-separated list of additional XID error codes to ignore + # Default ignored XIDs: 13, 31, 43, 45, 68, 109 (application errors) + additionalIgnoredXids: "" + # -- Enable XID event monitoring for health checks + healthCheckEnabled: true + +# -- NVML Provider Sidecar configuration +# Deploys the NVML provider as a sidecar container that connects to device-api-server +# via gRPC. This provides better isolation and independent updates compared to the +# built-in nvml provider. +nvmlProvider: + # -- Enable the NVML provider sidecar container + enabled: false + # -- Image configuration for the nvml-provider sidecar + image: + # -- Image repository + repository: ghcr.io/nvidia/nvsentinel/nvml-provider + # -- Image tag (defaults to Chart appVersion) + tag: "" + # -- Image pull policy + pullPolicy: IfNotPresent + # -- gRPC address of the provider service endpoint on device-api-server + # This should match the provider TCP listener (internal, not exposed externally) + serverAddress: "localhost:9001" + # -- Unique identifier for this provider instance + providerID: "nvml-provider-sidecar" + # -- Root path where NVIDIA driver libraries are located + driverRoot: /run/nvidia/driver + # -- Enable XID event monitoring for health checks + healthCheckEnabled: true + # -- HTTP port for health check endpoints + healthPort: 8082 + # -- Resource limits and requests for the sidecar + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + # -- Security context for the sidecar container + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + +# -- RuntimeClassName for the pod +# Set to "nvidia" when nvml.enabled is true to inject NVIDIA driver libraries +# Requires the NVIDIA GPU Operator or manual RuntimeClass configuration +runtimeClassName: "" + +# -- ServiceAccount configuration +serviceAccount: + # -- Create a ServiceAccount + create: true + # -- ServiceAccount name (generated if not set) + name: "" + # -- Annotations to add to the ServiceAccount + annotations: {} + # -- Automount service account token + automountServiceAccountToken: false + +# -- RBAC configuration +rbac: + # -- Create RBAC resources + create: true + +# -- Pod annotations +podAnnotations: {} + +# -- Pod labels +podLabels: {} + +# -- Pod security context +podSecurityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + +# -- Container security context +securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + +# -- Resource limits and requests +resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 256Mi + +# -- Node selector for scheduling +# @default -- Schedules only on GPU nodes +nodeSelector: + nvidia.com/gpu.present: "true" + +# -- Tolerations for scheduling +tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + +# -- Affinity rules +affinity: {} + +# -- Priority class name +priorityClassName: "" + +# -- Liveness probe configuration +livenessProbe: + httpGet: + path: /healthz + port: health + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +# -- Readiness probe configuration +readinessProbe: + httpGet: + path: /readyz + port: health + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +# -- Update strategy for the DaemonSet +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +# -- Service configuration (for metrics scraping) +service: + # -- Service type + type: ClusterIP + # -- Service annotations + annotations: {} + +# -- Prometheus metrics configuration +metrics: + # -- Enable metrics endpoint + enabled: true + # -- ServiceMonitor configuration (requires Prometheus Operator) + serviceMonitor: + # -- Create ServiceMonitor resource + enabled: false + # -- ServiceMonitor namespace (defaults to release namespace) + namespace: "" + # -- Additional labels for ServiceMonitor + labels: {} + # -- Scrape interval + interval: 30s + # -- Scrape timeout + scrapeTimeout: 10s + # -- Metric relabeling configs + metricRelabelings: [] + # -- Relabeling configs + relabelings: [] + # -- PrometheusRule configuration (requires Prometheus Operator) + prometheusRule: + # -- Create PrometheusRule resource + enabled: false + # -- PrometheusRule namespace (defaults to release namespace) + namespace: "" + # -- Additional labels for PrometheusRule + labels: {} + # -- Additional alerting rules + additionalRules: [] + +# -- Additional environment variables +env: [] +# - name: LOG_FORMAT +# value: json + +# -- Additional volume mounts +extraVolumeMounts: [] + +# -- Additional volumes +extraVolumes: [] + +# -- Init containers +initContainers: [] + +# -- Sidecar containers +sidecars: [] + +# -- HealthEvent CRD configuration +# Configures the HealthEvent custom resource for GPU health event tracking +# and coordination between NVSentinel components. +healthEvents: + # -- Enable HealthEvent CRD and controller + enabled: false + + # -- TTL (Time-To-Live) policy for automatic cleanup of resolved events + ttl: + # -- Duration after which resolved events are deleted + # Format: Go duration (e.g., "168h" = 7 days, "720h" = 30 days) + # Set to "0" to disable automatic cleanup + afterResolved: "168h" + + # -- Retention mode: "standard" or "compliance" + # - standard: Uses afterResolved for cleanup timing + # - compliance: Uses complianceRetention (overrides afterResolved) + retentionMode: "standard" + + # -- Retention period for compliance mode (e.g., "2160h" = 90 days) + complianceRetention: "2160h" + + # -- Maximum events to delete per cleanup cycle + batchSize: "100" + + # -- Interval between cleanup cycles + cleanupInterval: "5m" + + # -- Event history limits to prevent unbounded growth + limits: + # -- Maximum events per node (0 = unlimited) + maxEventsPerNode: "0" + + # -- Maximum total events cluster-wide (0 = unlimited) + maxTotalEvents: "0" diff --git a/deployments/helm/values-sidecar-test.yaml b/deployments/helm/values-sidecar-test.yaml new file mode 100644 index 000000000..a24811da3 --- /dev/null +++ b/deployments/helm/values-sidecar-test.yaml @@ -0,0 +1,56 @@ +# Sidecar test values - validates nvml-provider sidecar architecture +# Usage: helm upgrade nvsentinel deployments/helm/nvsentinel -n nvsentinel -f deployments/helm/values-sidecar-test.yaml + +image: + repository: ttl.sh/device-api-server-sidecar + tag: "2h" + pullPolicy: Always + +# Disable built-in NVML provider (use sidecar instead) +nvml: + enabled: false + +# Enable NVML Provider sidecar +nvmlProvider: + enabled: true + image: + repository: ttl.sh/nvml-provider-sidecar + tag: "2h" + pullPolicy: Always + serverAddress: "localhost:50051" + providerID: "nvml-provider-sidecar" + driverRoot: /run/nvidia/driver + healthCheckEnabled: true + healthPort: 8082 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + +# Override node selector (cluster uses node-type=gpu instead of nvidia.com/gpu.present) +# Set to null to remove the default, then add only the one we need +nodeSelector: + nvidia.com/gpu.present: null + node-type: gpu + +# RuntimeClass for NVML access +runtimeClassName: nvidia + +logging: + verbosity: 2 + format: json + +# Run as root to allow hostPath socket creation +podSecurityContext: + runAsNonRoot: false + runAsUser: 0 + runAsGroup: 0 + fsGroup: 0 + +securityContext: + runAsNonRoot: false + runAsUser: 0 + runAsGroup: 0 diff --git a/deployments/static/nvsentinel-daemonset.yaml b/deployments/static/nvsentinel-daemonset.yaml new file mode 100644 index 000000000..5fb9853a7 --- /dev/null +++ b/deployments/static/nvsentinel-daemonset.yaml @@ -0,0 +1,212 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NVSentinel Static Deployment Manifest +# +# This manifest deploys the Device API Server with the NVML Provider sidecar. +# For production use, consider using the Helm chart for better configurability. +# +# Usage: +# kubectl apply -f nvsentinel-daemonset.yaml +# +# Prerequisites: +# - Kubernetes 1.25+ +# - RuntimeClass 'nvidia' configured (GPU Operator or manual setup) +# - GPU nodes labeled with 'nvidia.com/gpu.present=true' + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: nvsentinel + labels: + app.kubernetes.io/name: nvsentinel +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nvsentinel + namespace: nvsentinel + labels: + app.kubernetes.io/name: nvsentinel +automountServiceAccountToken: false +--- +apiVersion: v1 +kind: Service +metadata: + name: nvsentinel + namespace: nvsentinel + labels: + app.kubernetes.io/name: nvsentinel +spec: + type: ClusterIP + clusterIP: None # Headless for DaemonSet + selector: + app.kubernetes.io/name: nvsentinel + ports: + - name: grpc + port: 50051 + targetPort: grpc + protocol: TCP + - name: metrics + port: 9090 + targetPort: metrics + protocol: TCP +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nvsentinel + namespace: nvsentinel + labels: + app.kubernetes.io/name: nvsentinel + app.kubernetes.io/component: device-api-server +spec: + selector: + matchLabels: + app.kubernetes.io/name: nvsentinel + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + app.kubernetes.io/name: nvsentinel + app.kubernetes.io/component: device-api-server + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: "/metrics" + spec: + serviceAccountName: nvsentinel + runtimeClassName: nvidia + nodeSelector: + nvidia.com/gpu.present: "true" + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + # Device API Server - Pure Go, no NVML dependencies + - name: device-api-server + image: ghcr.io/nvidia/nvsentinel/device-api-server:latest + imagePullPolicy: IfNotPresent + args: + - --grpc-address=127.0.0.1:50051 + - --health-port=8081 + - --metrics-port=9090 + - --provider-address=localhost:9001 + - --unix-socket=/var/run/device-api/device.sock + - --v=0 + ports: + - name: grpc + containerPort: 50051 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: health + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: health + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 256Mi + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: device-api-socket + mountPath: /var/run/device-api + + # NVML Provider Sidecar - CGO binary, requires RuntimeClass nvidia + - name: nvml-provider + image: ghcr.io/nvidia/nvsentinel/nvml-provider:latest + imagePullPolicy: IfNotPresent + args: + - --server-address=localhost:9001 + - --provider-id=nvml-provider + - --driver-root=/run/nvidia/driver + - --health-port=8082 + - --health-check-enabled=true + - --v=0 + ports: + - name: provider-health + containerPort: 8082 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: provider-health + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: provider-health + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumes: + - name: device-api-socket + emptyDir: {} diff --git a/docs/api/device-api-server.md b/docs/api/device-api-server.md new file mode 100644 index 000000000..b7725f6b4 --- /dev/null +++ b/docs/api/device-api-server.md @@ -0,0 +1,423 @@ +# Device API Server - API Reference + +This document provides the complete API reference for the Device API Server gRPC services. + +## Overview + +The Device API Server exposes a unified `GpuService` that provides both read and write operations following Kubernetes API conventions: + +| Operation Type | Methods | Clients | +|----------------|---------|---------| +| Read | `GetGpu`, `ListGpus`, `WatchGpus` | Consumers (device plugins, DRA drivers) | +| Write | `CreateGpu`, `UpdateGpu`, `UpdateGpuStatus`, `DeleteGpu` | Providers (health monitors, NVML) | + +**Package**: `nvidia.device.v1alpha1` + +**Connection Endpoints**: +- Unix Socket: `unix:///var/run/device-api/device.sock` (recommended) +- TCP: `localhost:50051` + +## GpuService + +The `GpuService` provides a unified API for GPU resource management: + +- **Read operations** (`GetGpu`, `ListGpus`, `WatchGpus`) for consumers +- **Write operations** (`CreateGpu`, `UpdateGpu`, `UpdateGpuStatus`, `DeleteGpu`) for providers + +> **Important**: Write operations acquire exclusive locks, blocking all consumer reads until completion. This prevents consumers from reading stale "healthy" states during GPU health transitions. + +### Read Operations + +### GetGpu + +Retrieves a single GPU resource by its unique name. + +```protobuf +rpc GetGpu(GetGpuRequest) returns (GetGpuResponse); +``` + +**Request**: + +| Field | Type | Description | +|-------|------|-------------| +| `name` | string | The unique resource name of the GPU | + +**Response**: + +| Field | Type | Description | +|-------|------|-------------| +| `gpu` | Gpu | The requested GPU resource | + +**Errors**: +- `NOT_FOUND`: GPU with the specified name does not exist + +**Example**: + +```bash +grpcurl -plaintext localhost:50051 \ + -d '{"name": "gpu-abc123"}' \ + nvidia.device.v1alpha1.GpuService/GetGpu +``` + +### ListGpus + +Retrieves a list of all GPU resources. + +```protobuf +rpc ListGpus(ListGpusRequest) returns (ListGpusResponse); +``` + +**Request**: Empty (reserved for future filtering/pagination) + +**Response**: + +| Field | Type | Description | +|-------|------|-------------| +| `gpu_list` | GpuList | List of all GPU resources | + +**Example**: + +```bash +grpcurl -plaintext localhost:50051 \ + nvidia.device.v1alpha1.GpuService/ListGpus +``` + +**Response Example**: + +```json +{ + "gpuList": { + "items": [ + { + "name": "gpu-abc123", + "spec": { + "uuid": "GPU-a1b2c3d4-e5f6-a7b8-c9d0-e1f2a3b4c5d6" + }, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "True", + "lastTransitionTime": "2026-01-21T10:00:00Z", + "reason": "GPUHealthy", + "message": "GPU is healthy and available" + } + ] + }, + "resourceVersion": "42" + } + ] + } +} +``` + +### WatchGpus + +Streams lifecycle events for GPU resources. The stream remains open until the client disconnects or an error occurs. + +```protobuf +rpc WatchGpus(WatchGpusRequest) returns (stream WatchGpusResponse); +``` + +**Request**: Empty (reserved for future filtering/resumption) + +**Response Stream**: + +| Field | Type | Description | +|-------|------|-------------| +| `type` | string | Event type: `ADDED`, `MODIFIED`, `DELETED`, `ERROR` | +| `object` | Gpu | The GPU resource (last known state for DELETED) | + +**Event Types**: + +| Type | Description | +|------|-------------| +| `ADDED` | GPU was registered or first observed | +| `MODIFIED` | GPU status was updated | +| `DELETED` | GPU was unregistered | +| `ERROR` | An error occurred in the watch stream | + +**Example**: + +```bash +grpcurl -plaintext localhost:50051 \ + nvidia.device.v1alpha1.GpuService/WatchGpus +``` + +**Behavior**: +- On connection, receives `ADDED` events for all existing GPUs +- Subsequent events reflect real-time changes +- Stream is per-client; multiple clients can watch simultaneously + +### Write Operations + +#### CreateGpu + +Creates a new GPU resource. This is the standard way for providers to register GPUs. + +```protobuf +rpc CreateGpu(CreateGpuRequest) returns (CreateGpuResponse); +``` + +**Request**: + +| Field | Type | Description | +|-------|------|-------------| +| `gpu` | Gpu | The GPU to create (metadata.name and spec.uuid required) | + +**Response**: + +| Field | Type | Description | +|-------|------|-------------| +| `gpu` | Gpu | The created GPU with server-assigned fields | +| `created` | bool | True if new GPU was created, false if already existed | + +**Errors**: +- `INVALID_ARGUMENT`: Required fields missing + +**Behavior**: +- If GPU already exists, returns existing GPU (idempotent) +- Triggers `ADDED` event for active watch streams + +**Example**: + +```bash +grpcurl -plaintext localhost:50051 \ + -d '{ + "gpu": { + "metadata": {"name": "gpu-abc123"}, + "spec": {"uuid": "GPU-a1b2c3d4-e5f6-a7b8-c9d0-e1f2a3b4c5d6"} + } + }' \ + nvidia.device.v1alpha1.GpuService/CreateGpu +``` + +#### UpdateGpu + +Replaces an entire GPU resource (spec and status). + +```protobuf +rpc UpdateGpu(UpdateGpuRequest) returns (Gpu); +``` + +**Request**: + +| Field | Type | Description | +|-------|------|-------------| +| `gpu` | Gpu | The GPU to update (metadata.name required) | + +**Response**: The updated GPU resource. + +**Errors**: +- `NOT_FOUND`: GPU does not exist +- `ABORTED`: Resource version conflict (optimistic concurrency) + +**Behavior**: +- Uses optimistic concurrency via `resource_version` +- Triggers `MODIFIED` event for active watch streams + +#### UpdateGpuStatus + +Updates only the status of an existing GPU (follows Kubernetes subresource pattern). + +```protobuf +rpc UpdateGpuStatus(UpdateGpuStatusRequest) returns (Gpu); +``` + +**Request**: + +| Field | Type | Description | +|-------|------|-------------| +| `name` | string | The GPU name to update | +| `status` | GpuStatus | New status (completely replaces existing) | +| `resource_version` | int64 | Optional: expected version for conflict detection | + +**Response**: The updated GPU resource. + +**Errors**: +- `NOT_FOUND`: GPU does not exist +- `ABORTED`: Resource version conflict (optimistic concurrency) + +**Locking**: Acquires exclusive write lock, blocking all reads. + +**Example** (mark GPU unhealthy due to XID error): + +```bash +grpcurl -plaintext localhost:50051 \ + -d '{ + "name": "gpu-abc123", + "status": { + "conditions": [{ + "type": "Ready", + "status": "False", + "reason": "XidError", + "message": "Critical XID error 79 detected" + }] + } + }' \ + nvidia.device.v1alpha1.GpuService/UpdateGpuStatus +``` + +#### DeleteGpu + +Removes a GPU from the server. + +```protobuf +rpc DeleteGpu(DeleteGpuRequest) returns (google.protobuf.Empty); +``` + +**Request**: + +| Field | Type | Description | +|-------|------|-------------| +| `name` | string | Unique identifier of GPU to remove | + +**Response**: Empty on success. + +**Errors**: +- `NOT_FOUND`: GPU does not exist + +**Behavior**: +- GPU will no longer appear in ListGpus/GetGpu responses +- Triggers `DELETED` event for active watch streams + +**Example**: + +```bash +grpcurl -plaintext localhost:50051 \ + -d '{"name": "gpu-abc123"}' \ + nvidia.device.v1alpha1.GpuService/DeleteGpu +``` + +--- + +## Resource Types + +### Gpu + +The main GPU resource following the Kubernetes Resource Model pattern. + +| Field | Type | Description | +|-------|------|-------------| +| `name` | string | Unique logical identifier | +| `spec` | GpuSpec | Identity and desired attributes | +| `status` | GpuStatus | Most recently observed state | +| `resource_version` | int64 | Monotonically increasing version | + +### GpuSpec + +Defines the identity of a GPU. + +| Field | Type | Description | +|-------|------|-------------| +| `uuid` | string | Physical hardware UUID (e.g., `GPU-a1b2c3d4-...`) | + +### GpuStatus + +Contains the observed state of a GPU. + +| Field | Type | Description | +|-------|------|-------------| +| `conditions` | Condition[] | Current state observations | +| `recommended_action` | string | Suggested resolution for negative states | + +### Condition + +Describes one aspect of the GPU's current state. + +| Field | Type | Description | +|-------|------|-------------| +| `type` | string | Category (e.g., `Ready`, `MemoryHealthy`) | +| `status` | string | `True`, `False`, or `Unknown` | +| `last_transition_time` | Timestamp | When status last changed | +| `reason` | string | Machine-readable reason (UpperCamelCase) | +| `message` | string | Human-readable details | + +**Standard Condition Types**: + +| Type | Description | +|------|-------------| +| `Ready` | Overall GPU health and availability | +| `MemoryHealthy` | GPU memory is functioning correctly | +| `ThermalHealthy` | GPU temperature is within safe limits | + +--- + +## Go Client Example + +```go +package main + +import ( + "context" + "log" + + v1alpha1 "github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +func main() { + // Connect via Unix socket (recommended) + conn, err := grpc.NewClient( + "unix:///var/run/device-api/device.sock", + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + log.Fatalf("failed to connect: %v", err) + } + defer conn.Close() + + client := v1alpha1.NewGpuServiceClient(conn) + + // Consumer: List GPUs + resp, err := client.ListGpus(context.Background(), &v1alpha1.ListGpusRequest{}) + if err != nil { + log.Fatalf("failed to list GPUs: %v", err) + } + + for _, gpu := range resp.GpuList.Items { + log.Printf("GPU: %s, Version: %d", gpu.Metadata.Name, gpu.Metadata.ResourceVersion) + for _, cond := range gpu.Status.Conditions { + log.Printf(" Condition: %s=%s (%s)", cond.Type, cond.Status, cond.Reason) + } + } + + // Provider: Update GPU status + _, err = client.UpdateGpuStatus(context.Background(), + &v1alpha1.UpdateGpuStatusRequest{ + Name: "gpu-abc123", + Status: &v1alpha1.GpuStatus{ + Conditions: []*v1alpha1.Condition{{ + Type: "Ready", + Status: "False", + Reason: "XidError", + Message: "Critical XID 79 detected", + }}, + }, + }) + if err != nil { + log.Fatalf("failed to update status: %v", err) + } +} +``` + +--- + +## Error Codes + +| Code | Meaning | +|------|---------| +| `NOT_FOUND` | GPU with specified name does not exist | +| `INVALID_ARGUMENT` | Request contains invalid parameters | +| `ABORTED` | Resource version conflict (optimistic concurrency) | +| `INTERNAL` | Server-side error occurred | +| `UNAVAILABLE` | Server is temporarily unavailable | + +--- + +## See Also + +- [Operations Guide](../operations/device-api-server.md) +- [Design Document](../design/device-api-server.md) +- [NVML Fallback Provider](../design/nvml-fallback-provider.md) diff --git a/docs/design/device-api-server.md b/docs/design/device-api-server.md new file mode 100644 index 000000000..89f159241 --- /dev/null +++ b/docs/design/device-api-server.md @@ -0,0 +1,695 @@ +# Device API Server - Design & Implementation Plan + +> **Status**: Draft +> **Author**: NVSentinel Team +> **Created**: 2026-01-21 + +## Table of Contents + +- [Executive Summary](#executive-summary) +- [Architecture Overview](#architecture-overview) +- [Design Decisions](#design-decisions) +- [Implementation Phases](#implementation-phases) +- [Directory Structure](#directory-structure) +- [API Design](#api-design) +- [Observability](#observability) +- [Deployment](#deployment) + +## Related Documents + +- [Implementation Tasks](./device-api-server-tasks.md) - Detailed task breakdown +- [NVML Fallback Provider](./nvml-fallback-provider.md) - Built-in NVML health provider design + +--- + +## Executive Summary + +The Device API Server is a **node-local gRPC cache server** deployed as a Kubernetes DaemonSet. It acts as an intermediary between: + +- **Providers** (e.g., NVSentinel health monitors) that update GPU device states +- **Consumers** (e.g., Device Plugins, DRA Drivers) that read device states for scheduling decisions + +### Key Requirements + +| Requirement | Description | +|-------------|-------------| +| Node-local | DaemonSet running on each GPU node | +| Read-blocking semantics | MUST block reads during provider updates to prevent stale data | +| Multiple providers | Support multiple health monitors updating different conditions | +| Multiple consumers | Support multiple readers (device-plugin, DRA driver, etc.) | +| Kubernetes patterns | klog/v2, structured logging, health probes | +| Helm-only deployment | No kustomize, pure Helm chart | +| Observability | Prometheus metrics, alerting rules | + +--- + +## Architecture Overview + +``` +┌─────────────────────────────────────────────────────────────────────────────────┐ +│ Kubernetes Node │ +├─────────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌──────────────────────┐ ┌──────────────────────────────┐ │ +│ │ NVSentinel │ │ Device Plugin / DRA │ │ +│ │ (Health Monitor) │ │ Driver │ │ +│ │ [Provider] │ │ [Consumer] │ │ +│ └──────────┬───────────┘ └──────────────┬───────────────┘ │ +│ │ │ │ +│ │ UpdateGpuStatus() │ GetGpu() │ +│ │ (gRPC) │ ListGpus() │ +│ │ │ WatchGpus() │ +│ ▼ ▼ │ +│ ┌──────────────────────────────────────────────────────────────────────────┐ │ +│ │ Device API Server (DaemonSet) │ │ +│ │ ┌────────────────────────────────────────────────────────────────────┐ │ │ +│ │ │ gRPC Server │ │ │ +│ │ │ ┌────────────────────────────────────────────────────────────┐ │ │ │ +│ │ │ │ GpuService (Unified) │ │ │ │ +│ │ │ │ Write: CreateGpu, UpdateGpu, UpdateGpuStatus, DeleteGpu │ │ │ │ +│ │ │ │ Read: GetGpu, ListGpus, WatchGpus │ │ │ │ +│ │ │ └────────────────────────────────┬───────────────────────────┘ │ │ │ +│ │ │ │ │ │ │ +│ │ │ ▼ │ │ │ +│ │ │ ┌─────────────────────────────────────────────────────────────┐ │ │ │ +│ │ │ │ Cache Layer │ │ │ │ +│ │ │ │ ┌───────────────────────────────────────────────────────┐ │ │ │ │ +│ │ │ │ │ sync.RWMutex (Writer-Preference) │ │ │ │ │ +│ │ │ │ │ │ │ │ │ │ +│ │ │ │ │ Write Lock() ──────────► Blocks ALL new RLock() │ │ │ │ │ +│ │ │ │ │ until write completes │ │ │ │ │ +│ │ │ │ │ │ │ │ │ │ +│ │ │ │ │ This ensures consumers NEVER read stale data when │ │ │ │ │ +│ │ │ │ │ a provider is updating (healthy → unhealthy) │ │ │ │ │ +│ │ │ │ └───────────────────────────────────────────────────────┘ │ │ │ │ +│ │ │ │ │ │ │ │ +│ │ │ │ ┌───────────────────────────────────────────────────────┐ │ │ │ │ +│ │ │ │ │ map[string]*Gpu (In-Memory Store) │ │ │ │ │ +│ │ │ │ └───────────────────────────────────────────────────────┘ │ │ │ │ +│ │ │ └─────────────────────────────────────────────────────────────┘ │ │ │ +│ │ │ │ │ │ +│ │ │ ┌─────────────────────────────────────────────────────────────┐ │ │ │ +│ │ │ │ Watch Broadcaster │ │ │ │ +│ │ │ │ Notifies all WatchGpus() streams on state changes │ │ │ │ +│ │ │ └─────────────────────────────────────────────────────────────┘ │ │ │ +│ │ └────────────────────────────────────────────────────────────────────┘ │ │ +│ │ │ │ +│ │ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────────────────┐ │ │ +│ │ │ Health │ │ Metrics │ │ Unix Socket │ │ │ +│ │ │ :8081 │ │ :9090 │ │ /var/run/device-api/device.sock │ │ │ +│ │ │ /healthz │ │ /metrics │ │ (node-local gRPC) │ │ │ +│ │ │ /readyz │ │ │ │ │ │ │ +│ │ └─────────────┘ └─────────────┘ └─────────────────────────────────┘ │ │ +│ └──────────────────────────────────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────────────┘ +``` + +### Data Flow: Read-Blocking Semantics + +``` +Timeline ──────────────────────────────────────────────────────────────────────────► + +Provider (NVSentinel) Cache (RWMutex) Consumer (Device Plugin) + │ │ │ + │ │◄──── RLock() ────────────────┤ GetGpu() + │ │ (allowed) │ + │ │──────────────────────────────►│ Returns data + │ │ RUnlock() │ + │ │ │ + │──── UpdateGpuStatus() ──────►│ │ + │ Lock() requested │ │ + │ │ │ + │ │◄──── RLock() ────────────────┤ GetGpu() + │ │ BLOCKED ⛔ │ (waits) + │ │ │ + │◄──── Lock() acquired ────────│ │ + │ (write in progress) │ │ + │ │ │ + │──── Update complete ────────►│ │ + │ Unlock() │ │ + │ │ │ + │ │──── RLock() allowed ─────────►│ + │ │ (fresh data) │ + │ │ │ + +⚠️ CRITICAL: Consumer NEVER reads stale "healthy" state when provider + is updating to "unhealthy". The RWMutex writer-preference ensures + new readers block once a write is pending. +``` + +--- + +## Design Decisions + +### D1: Read-Blocking vs Eventually Consistent + +| Option | Pros | Cons | Decision | +|--------|------|------|----------| +| **sync.RWMutex (writer-preference)** | Prevents stale reads; simple; Go-native | Readers blocked during writes | ✅ **Selected** | +| atomic.Value + copy-on-write | Never blocks readers | Readers may see stale data during update | ❌ Rejected | +| sync.Map | Good for read-heavy | No blocking semantics; may read stale | ❌ Rejected | + +**Rationale**: The requirement explicitly states "MUST block reads, preventing false positives when a node 'was' healthy, and the next state is unhealthy." This mandates write-blocking reads. + +### D2: Transport Protocol + +| Option | Pros | Cons | Decision | +|--------|------|------|----------| +| **Unix Socket** | Node-local only; no network exposure; fast | Pod must mount socket path | ✅ **Primary** | +| TCP localhost | Easy client setup | Requires port allocation | ✅ **Secondary** | +| hostNetwork + TCP | Accessible from host | Security risk | ❌ Rejected | + +**Rationale**: Unix socket provides security isolation and performance for node-local communication. TCP fallback for flexibility. + +### D3: Provider Registration Model + +| Option | Pros | Cons | Decision | +|--------|------|------|----------| +| **Implicit (any caller can update)** | Simple; stateless server | No provider identity tracking | ✅ **Phase 1** | +| Explicit registration | Track providers; detect failures | More complexity | 🔮 **Phase 2** | + +### D4: Logging Framework + +| Option | Pros | Cons | Decision | +|--------|------|------|----------| +| **klog/v2** | Kubernetes native; contextual logging; JSON format | Slightly verbose API | ✅ **Selected** | +| zap | Fast; popular | Not Kubernetes native | ❌ Rejected | +| logr | Interface-based | Needs backend anyway | Used via klog | + +--- + +## Implementation Phases + +### Phase 1: Core Server Foundation + +**Goal**: Minimal viable gRPC server with cache and blocking semantics. + +| Task ID | Task | Description | Estimate | +|---------|------|-------------|----------| +| P1.1 | Project scaffolding | Create `cmd/device-api-server/`, `internal/` structure | S | +| P1.2 | Proto extensions | Add provider-side RPCs (UpdateGpuStatus, RegisterGpu, UnregisterGpu) | M | +| P1.3 | Cache implementation | Thread-safe cache with RWMutex, writer-preference blocking | M | +| P1.4 | Consumer gRPC service | Implement GetGpu, ListGpus, WatchGpus (read path) | M | +| P1.5 | Provider gRPC service | Implement UpdateGpuStatus, RegisterGpu, UnregisterGpu (write path) | M | +| P1.6 | Watch broadcaster | Fan-out changes to all active WatchGpus streams | M | +| P1.7 | Graceful shutdown | SIGTERM handling, drain connections, health status | S | +| P1.8 | Unit tests | Cache tests, service tests, blocking behavior tests | L | + +**Deliverables**: +- Working gRPC server binary +- Consumer and Provider services +- Basic health endpoint + +--- + +### Phase 2: Kubernetes Integration + +**Goal**: Production-ready DaemonSet with proper k8s integration. + +| Task ID | Task | Description | Estimate | +|---------|------|-------------|----------| +| P2.1 | klog/v2 integration | Structured logging, contextual loggers, log levels | M | +| P2.2 | Health probes | gRPC health protocol, HTTP /healthz /readyz endpoints | M | +| P2.3 | Configuration | Flags, environment variables, config validation | S | +| P2.4 | Unix socket support | Listen on configurable socket path | S | +| P2.5 | Signal handling | Proper SIGTERM/SIGINT handling per k8s lifecycle | S | +| P2.6 | Integration tests | Test with mock providers/consumers | L | + +**Deliverables**: +- Kubernetes-ready binary +- Health endpoints +- Configurable via flags/env + +--- + +### Phase 3: Observability + +**Goal**: Full observability stack with metrics and alerts. + +| Task ID | Task | Description | Estimate | +|---------|------|-------------|----------| +| P3.1 | Prometheus metrics | Request counts, latencies, cache stats, connection counts | M | +| P3.2 | gRPC interceptors | grpc-prometheus interceptors for all RPCs | M | +| P3.3 | Custom metrics | `device_api_server_gpus_total`, `_unhealthy`, `_cache_*` | M | +| P3.4 | Metrics endpoint | HTTP /metrics on separate port | S | +| P3.5 | Alerting rules | PrometheusRule CRD for critical alerts | M | +| P3.6 | Grafana dashboard | JSON dashboard for visualization | M | + +**Metrics to implement**: + +``` +# Server metrics +device_api_server_info{version="...", go_version="..."} +device_api_server_up + +# Cache metrics +device_api_server_cache_gpus_total +device_api_server_cache_gpus_healthy +device_api_server_cache_gpus_unhealthy +device_api_server_cache_updates_total{provider="..."} +device_api_server_cache_lock_wait_seconds_bucket + +# gRPC metrics (via interceptor) +grpc_server_started_total{grpc_service, grpc_method} +grpc_server_handled_total{grpc_service, grpc_method, grpc_code} +grpc_server_handling_seconds_bucket{grpc_service, grpc_method} + +# Watch metrics +device_api_server_watch_streams_active +device_api_server_watch_events_total{type="ADDED|MODIFIED|DELETED"} +``` + +**Alerts**: + +```yaml +- alert: DeviceAPIServerDown + expr: up{job="device-api-server"} == 0 + for: 5m + +- alert: DeviceAPIServerHighLatency + expr: histogram_quantile(0.99, grpc_server_handling_seconds_bucket) > 0.5 + for: 5m + +- alert: DeviceAPIServerUnhealthyGPUs + expr: device_api_server_cache_gpus_unhealthy > 0 + for: 1m +``` + +--- + +### Phase 4: Helm Chart + +**Goal**: Production-ready Helm chart with all configurations. + +| Task ID | Task | Description | Estimate | +|---------|------|-------------|----------| +| P4.1 | Chart scaffolding | `charts/device-api-server/` structure | S | +| P4.2 | DaemonSet template | Node selector, tolerations, resource limits | M | +| P4.3 | RBAC templates | ServiceAccount, Role, RoleBinding | M | +| P4.4 | ConfigMap/Secret | Server configuration, TLS certs | M | +| P4.5 | Service templates | Headless service, metrics service | S | +| P4.6 | PrometheusRule | Alerting rules as k8s resource | M | +| P4.7 | ServiceMonitor | Prometheus scrape configuration | S | +| P4.8 | Values schema | JSON schema for values validation | M | +| P4.9 | Chart tests | Helm test hooks | M | +| P4.10 | Documentation | README, NOTES.txt, examples | M | + +**Chart Structure**: + +``` +charts/device-api-server/ +├── Chart.yaml +├── values.yaml +├── values.schema.json +├── README.md +├── templates/ +│ ├── _helpers.tpl +│ ├── daemonset.yaml +│ ├── serviceaccount.yaml +│ ├── role.yaml +│ ├── rolebinding.yaml +│ ├── configmap.yaml +│ ├── service.yaml +│ ├── service-metrics.yaml +│ ├── servicemonitor.yaml +│ ├── prometheusrule.yaml +│ ├── poddisruptionbudget.yaml +│ └── NOTES.txt +└── tests/ + └── test-connection.yaml +``` + +--- + +### Phase 5: Documentation & Polish + +**Goal**: Comprehensive documentation and production hardening. + +| Task ID | Task | Description | Estimate | +|---------|------|-------------|----------| +| P5.1 | Architecture docs | Design document, diagrams | M | +| P5.2 | API reference | Proto documentation, examples | M | +| P5.3 | Operations guide | Deployment, troubleshooting, runbooks | L | +| P5.4 | Developer guide | Contributing, local development | M | +| P5.5 | Security hardening | TLS, authentication review | M | +| P5.6 | Performance testing | Benchmark under load | L | +| P5.7 | CI/CD pipeline | GitHub Actions for build, test, release | M | + +--- + +## Directory Structure + +Following the [kubernetes-sigs/node-feature-discovery](https://github.com/kubernetes-sigs/node-feature-discovery) pattern +where the `api/` is a standalone module and `pkg/` contains public library code: + +``` +NVSentinel/ +├── api/ # STANDALONE API MODULE (own go.mod) +│ ├── gen/go/device/v1alpha1/ # Generated Go code +│ │ ├── gpu.pb.go +│ │ └── gpu_grpc.pb.go +│ ├── proto/device/v1alpha1/ # Proto definitions +│ │ └── gpu.proto # Unified GpuService (CRUD operations) +│ ├── go.mod # module github.com/nvidia/nvsentinel/api +│ ├── go.sum +│ └── Makefile +├── cmd/ # Command entry points (thin) +│ └── device-api-server/ +│ └── main.go # Server entrypoint only +├── pkg/ # PUBLIC LIBRARY CODE (importable) +│ ├── deviceapiserver/ # Device API Server implementation +│ │ ├── cache/ # Thread-safe GPU cache +│ │ │ ├── cache.go +│ │ │ ├── cache_test.go +│ │ │ └── broadcaster.go +│ │ ├── service/ # gRPC service implementation +│ │ │ └── gpu_service.go # GpuService (unified read/write) +│ │ ├── nvml/ # NVML provider (uses gRPC client) +│ │ │ ├── provider.go +│ │ │ ├── enumerator.go +│ │ │ └── health_monitor.go +│ │ ├── metrics/ # Prometheus metrics +│ │ └── health/ # Health check handlers +│ ├── version/ # Version information +│ │ └── version.go +│ └── signals/ # Signal handling utilities +├── charts/ # Helm charts +│ └── device-api-server/ +│ ├── Chart.yaml +│ ├── values.yaml +│ └── templates/ +├── docs/ +│ ├── design/ +│ ├── api/ +│ └── operations/ +├── hack/ # Build/development scripts +├── test/ # E2E tests +├── go.mod # Root module with replace directive +├── go.sum +└── Makefile +``` + +**Key Layout Decisions:** + +| Directory | Purpose | Importable | +|-----------|---------|------------| +| `api/` | Standalone API module for versioning | Yes (own module) | +| `pkg/` | Public library code | Yes | +| `cmd/` | Thin entry points | No | +| `charts/` | Helm deployment | N/A | + +Root `go.mod` uses: `replace github.com/nvidia/nvsentinel/api => ./api` + +--- + +## API Design + +### Unified GpuService + +Following Kubernetes API conventions, the API is consolidated into a single `GpuService` with standard CRUD methods: + +```protobuf +// GpuService provides a unified API for managing GPU resources. +// +// Read operations (Get, List, Watch) are intended for consumers. +// Write operations (Create, Update, UpdateStatus, Delete) are intended for providers. +service GpuService { + // Read Operations + rpc GetGpu(GetGpuRequest) returns (Gpu); + rpc ListGpus(ListGpusRequest) returns (ListGpusResponse); + rpc WatchGpus(WatchGpusRequest) returns (stream WatchGpusResponse); + + // Write Operations + rpc CreateGpu(CreateGpuRequest) returns (CreateGpuResponse); + rpc UpdateGpu(UpdateGpuRequest) returns (Gpu); + rpc UpdateGpuStatus(UpdateGpuStatusRequest) returns (Gpu); + rpc DeleteGpu(DeleteGpuRequest) returns (google.protobuf.Empty); +} + +message CreateGpuRequest { + Gpu gpu = 1; // metadata.name and spec.uuid required +} + +message CreateGpuResponse { + Gpu gpu = 1; + bool created = 2; // true if new, false if already existed +} + +message UpdateGpuRequest { + Gpu gpu = 1; // includes resource_version for optimistic concurrency +} + +message UpdateGpuStatusRequest { + string name = 1; + GpuStatus status = 2; + int64 resource_version = 3; // optional, for conflict detection +} + +message DeleteGpuRequest { + string name = 1; +} +``` + +**Design Rationale**: +- Single service simplifies API surface and tooling compatibility +- Standard CRUD verbs enable better integration with Kubernetes patterns +- `UpdateGpuStatus` follows the Kubernetes subresource pattern +- Optimistic concurrency via `resource_version` prevents lost updates + +--- + +## Observability + +### Metrics Architecture + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Device API Server │ +│ │ +│ ┌─────────────────────────────────────────────────────────┐ │ +│ │ gRPC Interceptors │ │ +│ │ grpc_server_started_total │ │ +│ │ grpc_server_handled_total │ │ +│ │ grpc_server_handling_seconds_bucket │ │ +│ └─────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌─────────────────────────────────────────────────────────┐ │ +│ │ Custom Metrics │ │ +│ │ device_api_server_cache_gpus_total │ │ +│ │ device_api_server_cache_lock_contention_total │ │ +│ │ device_api_server_watch_streams_active │ │ +│ └─────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌─────────────────────────────────────────────────────────┐ │ +│ │ Go Runtime Metrics │ │ +│ │ go_goroutines │ │ +│ │ go_memstats_alloc_bytes │ │ +│ │ process_cpu_seconds_total │ │ +│ └─────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ :9090/metrics │ +│ │ │ +└──────────────────────────────┼───────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ Prometheus │ +│ │ +│ ServiceMonitor ──► scrape_configs │ +│ │ +│ PrometheusRule ──► alerting_rules │ +│ │ +└─────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ Grafana │ +│ │ +│ Dashboard: Device API Server Overview │ +│ - Request rate / error rate │ +│ - P50/P99 latency │ +│ - GPU health summary │ +│ - Cache statistics │ +│ - Active watch streams │ +│ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Deployment + +### Helm Values (Key Configuration) + +```yaml +# values.yaml +replicaCount: 1 # DaemonSet ignores this, but kept for consistency + +image: + repository: ghcr.io/nvidia/device-api-server + tag: "" # Defaults to Chart appVersion + pullPolicy: IfNotPresent + +# Server configuration +server: + # gRPC listen address (TCP) - localhost only by default for security + # Set to ":50051" to bind to all interfaces (WARNING: unauthenticated API) + grpcAddress: "127.0.0.1:50051" + # Unix socket path (primary for node-local) + unixSocket: /var/run/device-api/device.sock + # Health probe port + healthPort: 8081 + # Metrics port + metricsPort: 9090 + +# Logging +logging: + # Log level (0=info, higher=more verbose) + verbosity: 0 + # Output format: text, json + format: json + +# Node selection +nodeSelector: + nvidia.com/gpu.present: "true" + +tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + +resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 256Mi + +# Security +securityContext: + runAsNonRoot: true + runAsUser: 65534 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + +# RBAC +serviceAccount: + create: true + name: "" + automountServiceAccountToken: false + +rbac: + create: true + +# Observability +metrics: + enabled: true + serviceMonitor: + enabled: true + interval: 30s + scrapeTimeout: 10s + prometheusRule: + enabled: true + +# Health probes +probes: + liveness: + initialDelaySeconds: 5 + periodSeconds: 10 + readiness: + initialDelaySeconds: 5 + periodSeconds: 10 +``` + +### DaemonSet Topology + +``` +┌─────────────────────────────────────────────────────────────────────────────────┐ +│ Kubernetes Cluster │ +├─────────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌───────────────────────┐ ┌───────────────────────┐ ┌───────────────────────┐│ +│ │ GPU Node 1 │ │ GPU Node 2 │ │ GPU Node 3 ││ +│ │ │ │ │ │ ││ +│ │ ┌─────────────────┐ │ │ ┌─────────────────┐ │ │ ┌─────────────────┐ ││ +│ │ │ device-api- │ │ │ │ device-api- │ │ │ │ device-api- │ ││ +│ │ │ server pod │ │ │ │ server pod │ │ │ │ server pod │ ││ +│ │ │ │ │ │ │ │ │ │ │ │ ││ +│ │ │ GPU-0: Healthy │ │ │ │ GPU-0: Healthy │ │ │ │ GPU-0: Unhealthy│ ││ +│ │ │ GPU-1: Healthy │ │ │ │ GPU-1: Healthy │ │ │ │ GPU-1: Healthy │ ││ +│ │ │ GPU-2: Healthy │ │ │ │ │ │ │ │ GPU-2: Healthy │ ││ +│ │ │ GPU-3: Healthy │ │ │ │ │ │ │ │ GPU-3: Healthy │ ││ +│ │ └─────────────────┘ │ │ └─────────────────┘ │ │ └─────────────────┘ ││ +│ │ │ │ │ │ ││ +│ │ /var/run/device-api/ │ │ /var/run/device-api/ │ │ /var/run/device-api/ ││ +│ │ device.sock │ │ device.sock │ │ device.sock ││ +│ │ │ │ │ │ ││ +│ └───────────────────────┘ └───────────────────────┘ └───────────────────────┘│ +│ │ +│ ┌───────────────────────┐ │ +│ │ Non-GPU Node │ (DaemonSet does NOT schedule here due to │ +│ │ (No GPU) │ nodeSelector: nvidia.com/gpu.present=true) │ +│ └───────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Risk Assessment + +| Risk | Impact | Likelihood | Mitigation | +|------|--------|------------|------------| +| Cache corruption on concurrent writes | High | Low | RWMutex provides exclusivity | +| Watch stream memory leak | Medium | Medium | Bounded channels, timeouts | +| Provider not updating (stale data) | High | Medium | Health checks, provider heartbeat (Phase 2) | +| Socket permission issues | Medium | Medium | Init container for socket dir | +| High lock contention | Medium | Low | Metrics to detect, sharding if needed | + +--- + +## Success Criteria + +### Phase 1 +- [ ] Server starts and accepts gRPC connections +- [ ] Provider can register/update/unregister GPUs +- [ ] Consumer can Get/List/Watch GPUs +- [ ] Read-blocking verified under concurrent load + +### Phase 2 +- [ ] Structured logs with klog/v2 +- [ ] Health probes pass in Kubernetes +- [ ] Unix socket communication works + +### Phase 3 +- [ ] Prometheus metrics exposed +- [ ] Grafana dashboard visualizes key metrics +- [ ] Alerts fire correctly in test scenarios + +### Phase 4 +- [ ] `helm install` works out of box +- [ ] DaemonSet schedules on GPU nodes only +- [ ] RBAC properly scoped + +### Phase 5 +- [ ] Documentation complete +- [ ] CI/CD pipeline green +- [ ] Performance benchmarks pass + +--- + +## Appendix: Research References + +1. **Kubernetes DaemonSet gRPC Best Practices** - Health probes, graceful shutdown, load balancing +2. **Go sync.RWMutex** - Writer-preference semantics, blocking behavior +3. **klog/v2** - Structured logging, contextual logging, JSON format +4. **Helm Chart Best Practices** - RBAC, ServiceAccount, DaemonSet templates +5. **grpc-prometheus** - Metrics interceptors, histogram configuration + +--- + +*Document version: 1.0* +*Last updated: 2026-01-21* diff --git a/docs/operations/device-api-server.md b/docs/operations/device-api-server.md new file mode 100644 index 000000000..f9a366084 --- /dev/null +++ b/docs/operations/device-api-server.md @@ -0,0 +1,446 @@ +# Device API Server - Operations Guide + +This guide covers deployment, configuration, monitoring, and troubleshooting of the Device API Server. + +## Architecture Overview + +``` +┌─────────────────────────────────────────────────────────────┐ +│ GPU Node │ +│ ┌─────────────────────────────────────────────────────────┐│ +│ │ Device API Server (DaemonSet) ││ +│ │ ┌─────────────────────────────────────────────────┐ ││ +│ │ │ GpuService (unified) │ ││ +│ │ │ Read: GetGpu, ListGpus, WatchGpus │ ││ +│ │ │ Write: CreateGpu, UpdateGpuStatus, DeleteGpu │ ││ +│ │ └────────────────────┬────────────────────────────┘ ││ +│ │ │ ││ +│ │ ▼ ││ +│ │ ┌─────────────────────────────────────────────────────┐││ +│ │ │ GPU Cache (RWMutex) │││ +│ │ │ - Read-blocking during writes │││ +│ │ │ - Watch event broadcasting │││ +│ │ └─────────────────────────────────────────────────────┘││ +│ │ ▲ ││ +│ │ │ (gRPC client) ││ +│ │ ┌────────────────────┴────────────────────────────┐ ││ +│ │ │ NVML Provider (optional) │ ││ +│ │ │ - GPU enumeration via CreateGpu │ ││ +│ │ │ - XID monitoring via UpdateGpuStatus │ ││ +│ │ └─────────────────────────────────────────────────┘ ││ +│ └─────────────────────────────────────────────────────────┘│ +│ │ +│ Clients: │ +│ - Device plugins (consumer) │ +│ - DRA drivers (consumer) │ +│ - External health monitors (provider) │ +└─────────────────────────────────────────────────────────────┘ +``` + +## Deployment + +### Prerequisites + +- Kubernetes 1.25+ +- Helm 3.0+ +- GPU nodes with label `nvidia.com/gpu.present=true` +- (Optional) NVIDIA GPU Operator for NVML provider +- (Optional) Prometheus Operator for monitoring + +### Installation + +**Basic Installation**: + +```bash +helm install device-api-server ./charts/device-api-server \ + --namespace device-api --create-namespace +``` + +**With NVML Provider**: + +```bash +helm install device-api-server ./charts/device-api-server \ + --namespace device-api --create-namespace \ + --set nvml.enabled=true +``` + +**With Prometheus Monitoring**: + +```bash +helm install device-api-server ./charts/device-api-server \ + --namespace device-api --create-namespace \ + --set metrics.serviceMonitor.enabled=true \ + --set metrics.prometheusRule.enabled=true +``` + +### Verify Installation + +```bash +# Check DaemonSet status +kubectl get daemonset -n device-api + +# Check pods are running on GPU nodes +kubectl get pods -n device-api -o wide + +# Check logs +kubectl logs -n device-api -l app.kubernetes.io/name=device-api-server +``` + +--- + +## Configuration + +### Command-Line Flags + +| Flag | Default | Description | +|------|---------|-------------| +| `--grpc-address` | `127.0.0.1:50051` | TCP address for gRPC server (localhost only by default) | +| `--unix-socket` | `/var/run/device-api/device.sock` | Unix socket path | +| `--health-port` | `8081` | HTTP port for health endpoints | +| `--metrics-port` | `9090` | HTTP port for Prometheus metrics | +| `--shutdown-timeout` | `30` | Graceful shutdown timeout (seconds) | +| `--shutdown-delay` | `5` | Pre-shutdown delay (seconds) | +| `--log-format` | `json` | Log format: `text` or `json` | +| `-v` | `0` | Log verbosity level | +| `--enable-nvml-provider` | `false` | Enable NVML fallback provider | +| `--nvml-driver-root` | `/run/nvidia/driver` | NVIDIA driver library path | +| `--nvml-health-check` | `true` | Enable XID event monitoring | +| `--nvml-ignored-xids` | `` | Additional XIDs to ignore (comma-separated) | + +### Helm Values + +See [charts/device-api-server/values.yaml](../../charts/device-api-server/values.yaml) for the complete reference. + +Key configuration sections: + +```yaml +# Server configuration +server: + grpcAddress: "127.0.0.1:50051" # localhost only by default for security + unixSocket: /var/run/device-api/device.sock + healthPort: 8081 + metricsPort: 9090 + +# NVML provider (built-in GPU discovery) +nvml: + enabled: false + driverRoot: /run/nvidia/driver + healthCheckEnabled: true + additionalIgnoredXids: "" + +# Node scheduling +nodeSelector: + nvidia.com/gpu.present: "true" + +# Resources +resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 256Mi +``` + +--- + +## NVML Provider + +The NVML provider enables built-in GPU enumeration and health monitoring without external providers. + +### How It Works + +1. **RuntimeClass**: Uses the `nvidia` RuntimeClass to inject NVIDIA driver libraries +2. **No GPU Consumption**: Sets `NVIDIA_DRIVER_CAPABILITIES=utility` to access NVML without consuming GPU resources +3. **XID Monitoring**: Listens for XID events and marks GPUs unhealthy for critical errors + +### Configuration + +```yaml +nvml: + enabled: true + driverRoot: /run/nvidia/driver # Container path to driver libs + healthCheckEnabled: true # Enable XID event monitoring + additionalIgnoredXids: "13,31" # Additional XIDs to ignore +``` + +### Default Ignored XIDs + +These XIDs are ignored by default (application errors, not hardware issues): + +| XID | Description | +|-----|-------------| +| 13 | Graphics Engine Exception | +| 31 | GPU memory page fault | +| 43 | GPU stopped processing | +| 45 | Preemptive cleanup | +| 68 | NVDEC0 Exception | +| 109 | Context switch timeout | + +### Critical XIDs (Mark GPU Unhealthy) + +| XID | Description | +|-----|-------------| +| 48 | Double-bit ECC error | +| 63 | ECC page retirement | +| 64 | ECC page retirement exceeded | +| 74 | NVLink error | +| 79 | GPU fallen off bus | +| 92 | High single-bit ECC error rate | +| 94 | GPU memory page retirement required | +| 95 | GPU memory page retirement failed | + +--- + +## Monitoring + +### Health Endpoints + +| Endpoint | Port | Description | +|----------|------|-------------| +| `/healthz` | 8081 | Liveness probe - server is running | +| `/readyz` | 8081 | Readiness probe - server is accepting traffic | +| `/metrics` | 9090 | Prometheus metrics | + +### Prometheus Metrics + +**Server Metrics**: + +| Metric | Type | Description | +|--------|------|-------------| +| `device_api_server_info` | Gauge | Server information (version, go_version) | + +**Cache Metrics**: + +| Metric | Type | Description | +|--------|------|-------------| +| `device_api_server_cache_gpus_total` | Gauge | Total GPUs in cache | +| `device_api_server_cache_gpus_healthy` | Gauge | Healthy GPUs | +| `device_api_server_cache_gpus_unhealthy` | Gauge | Unhealthy GPUs | +| `device_api_server_cache_gpus_unknown` | Gauge | GPUs with unknown status | +| `device_api_server_cache_updates_total` | Counter | Cache update operations | +| `device_api_server_cache_resource_version` | Gauge | Current cache version | + +**Watch Metrics**: + +| Metric | Type | Description | +|--------|------|-------------| +| `device_api_server_watch_streams_active` | Gauge | Active watch streams | +| `device_api_server_watch_events_total` | Counter | Watch events sent | + +**NVML Metrics**: + +| Metric | Type | Description | +|--------|------|-------------| +| `device_api_server_nvml_provider_enabled` | Gauge | NVML provider status | +| `device_api_server_nvml_gpu_count` | Gauge | GPUs discovered by NVML | +| `device_api_server_nvml_health_monitor_running` | Gauge | Health monitor status | + +### Alerting Rules + +When `metrics.prometheusRule.enabled=true`, the following alerts are created: + +| Alert | Severity | Condition | +|-------|----------|-----------| +| `DeviceAPIServerDown` | Critical | Server unreachable for 5m | +| `DeviceAPIServerHighLatency` | Warning | P99 latency > 500ms | +| `DeviceAPIServerHighErrorRate` | Warning | Error rate > 10% | +| `DeviceAPIServerUnhealthyGPUs` | Warning | Unhealthy GPUs > 0 | +| `DeviceAPIServerNoGPUs` | Warning | No GPUs for 10m | +| `DeviceAPIServerNVMLProviderDown` | Warning | NVML provider not running | +| `DeviceAPIServerNVMLHealthMonitorDown` | Warning | Health monitor not running | +| `DeviceAPIServerHighMemory` | Warning | Memory > 512MB | + +### Grafana Dashboard + +Example PromQL queries for dashboards: + +```promql +# GPU health overview +device_api_server_cache_gpus_healthy / device_api_server_cache_gpus_total * 100 + +# Watch stream activity +rate(device_api_server_watch_events_total[5m]) + +# Cache update rate +rate(device_api_server_cache_updates_total[5m]) +``` + +--- + +## Troubleshooting + +### Pod Not Scheduling + +**Symptom**: DaemonSet shows 0/N pods ready + +**Check**: + +```bash +# Verify node labels +kubectl get nodes --show-labels | grep gpu + +# Check DaemonSet events +kubectl describe daemonset -n device-api device-api-server +``` + +**Solution**: Ensure nodes have `nvidia.com/gpu.present=true` label or override `nodeSelector`. + +### NVML Provider Fails to Start + +**Symptom**: Logs show `NVML initialization failed` + +**Check**: + +```bash +# Verify RuntimeClass exists +kubectl get runtimeclass nvidia + +# Check NVIDIA driver on node +kubectl debug node/ -it --image=nvidia/cuda:12.0-base -- nvidia-smi +``` + +**Solutions**: +1. Install NVIDIA GPU Operator +2. Manually create `nvidia` RuntimeClass +3. Verify driver installation on nodes + +### Permission Denied on Unix Socket + +**Symptom**: Clients cannot connect to Unix socket + +**Check**: + +```bash +# Check socket permissions on node +ls -la /var/run/device-api/ +``` + +**Solution**: Verify `securityContext` allows socket creation, or adjust `runAsUser`. + +### GPUs Not Appearing + +**Symptom**: `ListGpus` returns empty + +**Check**: + +```bash +# Check if NVML provider is enabled +kubectl logs -n device-api | grep nvml + +# Check for GPU enumeration errors +kubectl logs -n device-api | grep -i error +``` + +**Solutions**: +1. Enable NVML provider: `--set nvml.enabled=true` +2. Deploy an external health provider +3. Check NVML can access GPUs + +### High Memory Usage + +**Symptom**: Pod OOMKilled or memory alerts firing + +**Check**: + +```bash +# Check current memory usage +kubectl top pods -n device-api + +# Check watch stream count +curl -s http://:9090/metrics | grep watch_streams +``` + +**Solutions**: +1. Increase memory limits +2. Investigate clients creating excessive watch streams +3. Check for memory leaks in logs + +### Watch Stream Disconnections + +**Symptom**: Consumers report frequent reconnections + +**Check**: + +```bash +# Check network policy +kubectl get networkpolicy -n device-api + +# Check for errors in logs +kubectl logs -n device-api | grep -i "stream\|watch" +``` + +**Solutions**: +1. Ensure network policies allow intra-node traffic +2. Check client timeout settings +3. Verify server is not overloaded + +--- + +## Graceful Shutdown + +The server implements graceful shutdown: + +1. **PreStop Hook**: Sleeps for `shutdownDelay` seconds +2. **Signal Handling**: Catches SIGTERM/SIGINT +3. **Drain Period**: Stops accepting new connections +4. **In-Flight Completion**: Waits for active requests (up to `shutdownTimeout`) +5. **Resource Cleanup**: Closes connections, stops NVML + +**Timeline**: + +``` +SIGTERM → [shutdownDelay] → Stop listeners → [shutdownTimeout] → Force close +``` + +Configure in Helm: + +```yaml +server: + shutdownTimeout: 30 # Max wait for in-flight requests + shutdownDelay: 5 # Pre-shutdown delay for endpoint propagation +``` + +--- + +## Security Considerations + +### Pod Security + +Default security context (non-root, restricted): + +```yaml +securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL +``` + +### Network Security + +> **Warning**: The gRPC API is unauthenticated. Take care when exposing beyond localhost. + +- gRPC over TCP is **plaintext and unauthenticated** by default and is intended **only for node-local access**. +- The TCP listener binds to `127.0.0.1:50051` (localhost) by default. This prevents network exposure. +- Prefer using a Unix domain socket for all local clients and providers; avoid exposing the TCP gRPC listener to the broader cluster network. +- In multi-tenant or partially untrusted clusters, strongly recommend **disabling the TCP listener** (set `--grpc-address=""`) or using a restricted Unix socket combined with Kubernetes `NetworkPolicy` to limit access to the Device API Server pod. +- If TCP gRPC must be exposed beyond the node (e.g., `--grpc-address=:50051`), terminate it behind TLS/mTLS or an equivalent authenticated tunnel (for example, a sidecar proxy or ingress with client auth) and enforce a least-privilege `NetworkPolicy`. + +### Service Account + +- `automountServiceAccountToken: false` by default +- No Kubernetes API access required + +--- + +## See Also + +- [API Reference](../api/device-api-server.md) +- [Design Document](../design/device-api-server.md) +- [NVML Fallback Provider](../design/nvml-fallback-provider.md) +- [Helm Chart README](../../charts/device-api-server/README.md) diff --git a/docs/plans/2026-02-04-hybrid-device-apiserver-design.md b/docs/plans/2026-02-04-hybrid-device-apiserver-design.md new file mode 100644 index 000000000..f8efdbae7 --- /dev/null +++ b/docs/plans/2026-02-04-hybrid-device-apiserver-design.md @@ -0,0 +1,455 @@ +# Hybrid Device API Server Design + +**Date:** 2026-02-04 +**Status:** Approved for implementation +**Base:** PR #718 (merged) + PR #720 (to cherry-pick) +**Authors:** @pteranodan (PR #718), @ArangoGutierrez (PR #720) + +--- + +## Executive Summary + +This design combines PR #718's architectural foundation with PR #720's runtime components, replacing Kine/SQLite persistent storage with an in-memory cache. GPU state is ephemeral by design—providers are the source of truth. + +**Key decisions:** +1. **In-memory cache** - No persistence. On restart, providers re-register GPUs. +2. **Readiness gate** - Server blocks consumer requests until providers register (Option A). +3. **Memory bounds** - Sized for Vera Rubin Ultra NVL576 (576 GPUs, 2027). +4. **Preserve authorship** - All commits include `Co-authored-by` for both authors. + +--- + +## Why In-Memory (Not Persistent) + +Persistent storage is harmful for GPU status because: + +1. **Stale data is dangerous** - After restart, serving cached "Ready: True" when GPU is actually bad could cause workload failures. +2. **Providers are source of truth** - NVML, DCGM, NVSentinel health monitors own GPU state. +3. **Cleaner recovery** - Restart → empty → providers re-register → fresh state. +4. **Simpler failure modes** - No SQLite corruption, no compaction, no "database locked." + +--- + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Device API Server │ +│ │ +│ ┌─────────────────────────────────────────────────────────────┐│ +│ │ gRPC Server ││ +│ │ ┌─────────────┐ ┌─────────────┐ ┌──────────────────┐ ││ +│ │ │ GpuService │ │HealthProbe │ │ Metrics/Reflect │ ││ +│ │ └──────┬──────┘ └──────┬──────┘ └──────────────────┘ ││ +│ └─────────┼────────────────┼──────────────────────────────────┘│ +│ │ │ │ +│ ▼ ▼ │ +│ ┌─────────────────────────────────────────────────────────────┐│ +│ │ In-Memory Cache ││ +│ │ ┌──────────────┐ ┌──────────────┐ ┌─────────────────┐ ││ +│ │ │ GPU Registry │ │ Broadcaster │ │ Readiness Gate │ ││ +│ │ │ (RWMutex) │ │ (Watch) │ │ (provider count)│ ││ +│ │ └──────────────┘ └──────────────┘ └─────────────────┘ ││ +│ └─────────────────────────────────────────────────────────────┘│ +│ ▲ │ +│ ┌───────────────┼───────────────┐ │ +│ │ │ │ │ +│ ┌───────────┴───┐ ┌───────┴───────┐ ┌────┴────────┐ │ +│ │ NVML Provider │ │ External Prov │ │ Future Prov │ │ +│ │ (built-in) │ │ (NVSentinel) │ │ (DCGM) │ │ +│ └───────────────┘ └───────────────┘ └─────────────┘ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Memory Safety Bounds + +Based on research of NVIDIA architectures: + +| System | Year | GPUs per Unit | +|--------|------|---------------| +| DGX H100 | 2022 | 8 | +| DGX B200 | 2024 | 8 | +| GB200 NVL72 | 2024 | 72 | +| Vera Rubin NVL72 | 2026 | 72 | +| Vera Rubin NVL144 | 2026 | 144 | +| Vera Rubin Ultra NVL576 | 2027 | **576** | + +### CacheLimits Configuration + +```go +type CacheLimits struct { + // GPU registration limits + MaxGPUs int // Default: 1024 (covers NVL576 + headroom) + MaxConditionsPerGPU int // Default: 32 + MaxLabelsPerGPU int // Default: 64 + MaxGPUObjectBytes int // Default: 64KB per GPU object + + // Watch subscriber limits + MaxSubscribers int // Default: 256 + SubscriberRateLimit rate.Limit // Default: 10 subscribes/sec + SubscriberBurst int // Default: 20 + + // Per-subscriber limits + EventBufferSize int // Default: 256 events + SlowSubscriberTimeout time.Duration // Default: 30s, then disconnect +} + +func DefaultCacheLimits() CacheLimits { + return CacheLimits{ + MaxGPUs: 1024, + MaxConditionsPerGPU: 32, + MaxLabelsPerGPU: 64, + MaxGPUObjectBytes: 64 * 1024, + MaxSubscribers: 256, + SubscriberRateLimit: rate.Limit(10), + SubscriberBurst: 20, + EventBufferSize: 256, + SlowSubscriberTimeout: 30 * time.Second, + } +} +``` + +### Enforcement Points + +| Limit | Enforced At | Behavior on Exceed | +|-------|-------------|-------------------| +| `MaxGPUs` | `CreateGpu()` | Return `ResourceExhausted` | +| `MaxConditionsPerGPU` | `UpdateGpuStatus()` | Truncate + log warning | +| `MaxGPUObjectBytes` | All writes | Return `InvalidArgument` | +| `MaxSubscribers` | `Subscribe()` | Return `ResourceExhausted` | +| `SubscriberRateLimit` | `Subscribe()` | Return `ResourceExhausted` with retry-after | +| `SlowSubscriberTimeout` | Broadcaster goroutine | Disconnect + log | + +### Memory Budget at Max Scale (NVL576) + +| Component | Calculation | Max Memory | +|-----------|-------------|------------| +| GPU objects | 576 GPUs × 64KB | ~37 MB | +| Watch buffers | 256 subscribers × 256 events × ~1KB | ~65 MB | +| Overhead | ~20% | ~20 MB | +| **Total** | | **~122 MB** | + +Container limit of 256MB provides 2x headroom. + +--- + +## Readiness Gate + +Server blocks consumer requests until providers register GPUs. + +### Startup Sequence + +``` +STARTING → WAITING_FOR_PROVIDERS → READY → SERVING + │ │ │ │ + gRPC up Health: Health: Health: + Cache empty NOT_SERVING SERVING SERVING +``` + +### Readiness Conditions + +```go +type ReadinessGate struct { + mu sync.RWMutex + + minGPUs int // Default: 1 + providerTimeout time.Duration // Default: 30s + + ready bool + readySince time.Time + registeredGPUs int + activeProviders map[string]providerState +} + +func (r *ReadinessGate) checkReadiness() { + // Condition 1: At least minGPUs registered + if r.registeredGPUs >= r.minGPUs { + r.markReady("gpu_threshold_met") + return + } + + // Condition 2: Timeout expired (node might have no GPUs) + if time.Since(r.startTime) > r.providerTimeout { + r.markReady("provider_timeout") + return + } +} +``` + +### Health Probe Behavior + +| Endpoint | Before Ready | After Ready | +|----------|--------------|-------------| +| `/healthz` (liveness) | `200 OK` | `200 OK` | +| `/readyz` (readiness) | `503 Service Unavailable` | `200 OK` | +| gRPC `Health/Check` | `NOT_SERVING` | `SERVING` | + +### Interceptor + +```go +func (s *Server) readinessInterceptor(ctx context.Context, req interface{}, + info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + + // Allow provider writes always + if isProviderMethod(info.FullMethod) { + return handler(ctx, req) + } + + // Block consumer reads until ready + if !s.readinessGate.IsReady() { + return nil, status.Error(codes.Unavailable, + "server initializing, waiting for GPU providers") + } + + return handler(ctx, req) +} +``` + +--- + +## Watch Broadcaster + +Non-blocking fan-out to multiple subscribers with slow subscriber eviction. + +```go +type Broadcaster struct { + mu sync.RWMutex + subscribers map[string]*subscriber + limits CacheLimits + logger klog.Logger + metrics *BroadcasterMetrics + subscribeLimiter *rate.Limiter +} + +type subscriber struct { + id string + channel chan WatchEvent + createdAt time.Time + lastSend time.Time + dropCount int64 +} + +// Non-blocking broadcast - drops events for slow subscribers +func (b *Broadcaster) Broadcast(event WatchEvent) { + b.mu.RLock() + defer b.mu.RUnlock() + + for id, sub := range b.subscribers { + select { + case sub.channel <- event: + sub.lastSend = time.Now() + default: + atomic.AddInt64(&sub.dropCount, 1) + b.metrics.EventsDropped.WithLabelValues(id).Inc() + } + } +} + +// Periodic eviction of stuck subscribers +func (b *Broadcaster) evictSlowSubscribers(ctx context.Context) { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + b.mu.Lock() + now := time.Now() + for id, sub := range b.subscribers { + if now.Sub(sub.lastSend) > b.limits.SlowSubscriberTimeout { + close(sub.channel) + delete(b.subscribers, id) + b.metrics.SubscribersEvicted.Inc() + } + } + b.mu.Unlock() + } + } +} +``` + +--- + +## What to Take from Each PR + +### From PR #718 (pteranodan) ✅ + +- Single unified Go module +- `internal/generated/` proto path +- Options → Config → CompletedConfig pattern +- `pkg/client-go/` generated clients +- Service registry pattern +- `errgroup` startup/shutdown +- Examples + +### From PR #720 (ArangoGutierrez) ✅ + +- In-memory cache with RWMutex +- Watch Broadcaster +- NVML Provider +- XID health monitoring +- Helm chart +- Dockerfile (multi-stage) + +### Remove ❌ + +- Kine/SQLite storage (from #718) +- Multi-module layout (from #720) +- `api/gen/go/` proto path (from #720) + +### New Components (Neither PR) + +- CacheLimits with memory bounds +- ReadinessGate +- Slow subscriber eviction +- Provider heartbeat (optional) + +--- + +## Package Structure + +``` +github.com/nvidia/nvsentinel/ +├── api/ +│ ├── device/v1alpha1/ # Go types +│ └── proto/ # Proto definitions +├── cmd/ +│ ├── device-apiserver/ # Main binary +│ └── nvml-provider/ # NVML sidecar +├── internal/ +│ └── generated/ # Proto output +├── pkg/ +│ ├── apiserver/ +│ │ ├── cache/ # In-memory cache +│ │ ├── broadcaster/ # Watch fan-out +│ │ ├── config/ # Options pattern +│ │ ├── metrics/ # Prometheus +│ │ ├── registry/ # Service registry +│ │ └── server.go # Main server +│ ├── client-go/ # Generated clients +│ ├── grpc/client/ # Client helpers +│ └── nvml/ # NVML provider +├── deployments/ +│ ├── helm/ # Helm chart +│ └── container/ # Dockerfile +└── examples/ +``` + +--- + +## Metrics + +``` +# Cache operations +device_api_cache_gpus_current +device_api_cache_bytes_current +device_api_cache_rejected_total{reason="max_gpus_exceeded|object_too_large"} +device_api_cache_truncations_total{field="conditions"} + +# Broadcaster +device_api_broadcaster_subscribers_current +device_api_broadcaster_subscribers_evicted_total +device_api_broadcaster_subscriptions_rejected_total{reason="rate_limited|max_exceeded"} +device_api_broadcaster_events_sent_total +device_api_broadcaster_events_dropped_total{subscriber="..."} + +# Readiness +device_api_readiness_state{state="waiting|ready"} +device_api_readiness_transition_timestamp +``` + +--- + +## Implementation Phases + +### Phase 1: Storage Swap +- Remove Kine/SQLite dependencies from #718 +- Add in-memory cache (from #720) +- Add CacheLimits +- Add Broadcaster +- Wire cache into server + +### Phase 2: Readiness Gate +- Add ReadinessGate +- Add readiness interceptor +- Update health probes +- Add provider timeout fallback + +### Phase 3: NVML Provider +- Port `pkg/nvml/` from #720 +- Port XID health monitoring +- Integrate with cache +- Add build tags for CGO + +### Phase 4: Deployment +- Port Helm chart from #720 +- Port Dockerfile +- Update CI workflow +- Add integration tests + +### Phase 5: Documentation +- Update README +- Add architecture doc +- Add operations guide + +--- + +## Authorship Requirements + +**CRITICAL:** Every commit MUST include both authors: + +``` +feat: implement in-memory GPU cache with memory bounds + +Co-authored-by: Carlos Eduardo Arango Gutierrez +Co-authored-by: Dan Stone +``` + +### PR Description Template + +```markdown +## Summary + +Hybrid Device API Server combining #718 and #720. + +**Key changes from original PRs:** +- Replaced Kine/SQLite with in-memory cache +- Added memory safety bounds (MaxGPUs: 1024 for Rubin Ultra NVL576) +- Added readiness gate (blocks until providers register) + +## Attribution + +This work is based on contributions from: +- @pteranodan - PR #718: Core architecture, client generation, options pattern +- @ArangoGutierrez - PR #720: In-memory cache, NVML provider, Helm chart + +All commits include `Co-authored-by` trailers. + +## References +- Closes #720 (PR #718 already merged) +``` + +--- + +## Quick Reference for Next Session + +1. **PR #718 is merged** - Base is ready +2. **Rebase #720 on main** - Will have conflicts +3. **Remove from #720:** `api/go.mod`, `client-go/` changes, `api/gen/go/` references +4. **Update imports:** `api/gen/go/` → `internal/generated/` +5. **Add new files:** CacheLimits, ReadinessGate, slow subscriber eviction +6. **Port from #720:** `pkg/deviceapiserver/cache/`, `pkg/deviceapiserver/nvml/`, Helm, Dockerfile +7. **Reconcile protos:** Merge both sets of changes, regenerate +8. **Every commit:** Include both `Co-authored-by` trailers + +--- + +## References + +- [NVIDIA GB200 NVL72](https://www.nvidia.com/en-us/data-center/gb200-nvl72/) +- [Vera Rubin Platform - Tom's Hardware](https://www.tomshardware.com/pc-components/gpus/nvidias-vera-rubin-platform-in-depth-inside-nvidias-most-complex-ai-and-hpc-platform-to-date) +- [Rubin Ultra 576 GPUs - TweakTown](https://www.tweaktown.com/news/108238/nvidia-teases-next-gen-kyber-rack-scale-tech-up-to-576-nvidia-rubin-ultra-gpus-in-2027/index.html) diff --git a/go.mod b/go.mod new file mode 100644 index 000000000..7fa5c8739 --- /dev/null +++ b/go.mod @@ -0,0 +1,75 @@ +module github.com/nvidia/nvsentinel + +go 1.25.0 + +require ( + github.com/NVIDIA/go-nvml v0.12.9-0 + github.com/google/uuid v1.6.0 + github.com/nvidia/nvsentinel/api v0.0.0-00010101000000-000000000000 + github.com/prometheus/client_golang v1.23.2 + google.golang.org/grpc v1.77.0 + google.golang.org/protobuf v1.36.10 + k8s.io/api v0.34.1 + k8s.io/apimachinery v0.35.0 + k8s.io/client-go v0.34.1 + k8s.io/klog/v2 v2.130.1 + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 + sigs.k8s.io/controller-runtime v0.22.0 +) + +require ( + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/zapr v1.3.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + github.com/spf13/pflag v1.0.9 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/oauth2 v0.32.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/term v0.37.0 // indirect + golang.org/x/text v0.31.0 // indirect + golang.org/x/time v0.9.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251124214823-79d6a2a48846 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.34.0 // indirect + k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect +) + +replace github.com/nvidia/nvsentinel/api => ./api diff --git a/go.sum b/go.sum new file mode 100644 index 000000000..aceaae25a --- /dev/null +++ b/go.sum @@ -0,0 +1,228 @@ +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/NVIDIA/go-nvml v0.12.9-0 h1:e344UK8ZkeMeeLkdQtRhmXRxNf+u532LDZPGMtkdus0= +github.com/NVIDIA/go-nvml v0.12.9-0/go.mod h1:+KNA7c7gIBH7SKSJ1ntlwkfN80zdx8ovl4hrK3LmPt4= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= +github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= +github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= +github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= +github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= +golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251124214823-79d6a2a48846 h1:Wgl1rcDNThT+Zn47YyCXOXyX/COgMTIdhJ717F0l4xk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251124214823-79d6a2a48846/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM= +google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.34.1 h1:jC+153630BMdlFukegoEL8E/yT7aLyQkIVuwhmwDgJM= +k8s.io/api v0.34.1/go.mod h1:SB80FxFtXn5/gwzCoN6QCtPD7Vbu5w2n1S0J5gFfTYk= +k8s.io/apiextensions-apiserver v0.34.0 h1:B3hiB32jV7BcyKcMU5fDaDxk882YrJ1KU+ZSkA9Qxoc= +k8s.io/apiextensions-apiserver v0.34.0/go.mod h1:hLI4GxE1BDBy9adJKxUxCEHBGZtGfIg98Q+JmTD7+g0= +k8s.io/apimachinery v0.35.0 h1:Z2L3IHvPVv/MJ7xRxHEtk6GoJElaAqDCCU0S6ncYok8= +k8s.io/apimachinery v0.35.0/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= +k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY= +k8s.io/client-go v0.34.1/go.mod h1:kA8v0FP+tk6sZA0yKLRG67LWjqufAoSHA2xVGKw9Of8= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.22.0 h1:mTOfibb8Hxwpx3xEkR56i7xSjB+nH4hZG37SrlCY5e0= +sigs.k8s.io/controller-runtime v0.22.0/go.mod h1:FwiwRjkRPbiN+zp2QRp7wlTCzbUXxZ/D4OzuQUDwBHY= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/pkg/controllers/healthevents/conditions.go b/pkg/controllers/healthevents/conditions.go new file mode 100644 index 000000000..c5c9deba8 --- /dev/null +++ b/pkg/controllers/healthevents/conditions.go @@ -0,0 +1,115 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package healthevents + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + nvsentinelv1alpha1 "github.com/nvidia/nvsentinel/api/nvsentinel/v1alpha1" +) + +// SetCondition sets or updates a condition in the conditions slice. +// If the condition already exists, it updates the existing one. +// If the condition doesn't exist, it appends a new one. +func SetCondition(conditions *[]nvsentinelv1alpha1.HealthEventCondition, condition nvsentinelv1alpha1.HealthEventCondition) { + if conditions == nil { + return + } + + // Find existing condition + for i, existing := range *conditions { + if existing.Type == condition.Type { + // Only update if something changed + if existing.Status != condition.Status || + existing.Reason != condition.Reason || + existing.Message != condition.Message { + (*conditions)[i] = condition + } + return + } + } + + // Condition not found, append + *conditions = append(*conditions, condition) +} + +// GetCondition returns the condition with the specified type, or nil if not found. +func GetCondition(conditions []nvsentinelv1alpha1.HealthEventCondition, conditionType nvsentinelv1alpha1.HealthEventConditionType) *nvsentinelv1alpha1.HealthEventCondition { + for i := range conditions { + if conditions[i].Type == conditionType { + return &conditions[i] + } + } + return nil +} + +// IsConditionTrue returns true if the condition with the specified type is True. +func IsConditionTrue(conditions []nvsentinelv1alpha1.HealthEventCondition, conditionType nvsentinelv1alpha1.HealthEventConditionType) bool { + condition := GetCondition(conditions, conditionType) + return condition != nil && condition.Status == metav1.ConditionTrue +} + +// IsConditionFalse returns true if the condition with the specified type is False. +func IsConditionFalse(conditions []nvsentinelv1alpha1.HealthEventCondition, conditionType nvsentinelv1alpha1.HealthEventConditionType) bool { + condition := GetCondition(conditions, conditionType) + return condition != nil && condition.Status == metav1.ConditionFalse +} + +// RemoveCondition removes the condition with the specified type. +func RemoveCondition(conditions *[]nvsentinelv1alpha1.HealthEventCondition, conditionType nvsentinelv1alpha1.HealthEventConditionType) { + if conditions == nil { + return + } + + newConditions := make([]nvsentinelv1alpha1.HealthEventCondition, 0, len(*conditions)) + for _, c := range *conditions { + if c.Type != conditionType { + newConditions = append(newConditions, c) + } + } + *conditions = newConditions +} + +// InitializeConditions sets up the initial conditions for a new HealthEvent. +func InitializeConditions(healthEvent *nvsentinelv1alpha1.HealthEvent) { + now := metav1.Now() + + // Set Detected condition + SetCondition(&healthEvent.Status.Conditions, nvsentinelv1alpha1.HealthEventCondition{ + Type: nvsentinelv1alpha1.ConditionDetected, + Status: metav1.ConditionTrue, + LastTransitionTime: now, + Reason: "EventCreated", + Message: "Health event detected and recorded", + ObservedGeneration: healthEvent.Generation, + }) + + // Initialize other conditions as Unknown + for _, condType := range []nvsentinelv1alpha1.HealthEventConditionType{ + nvsentinelv1alpha1.ConditionNodeQuarantined, + nvsentinelv1alpha1.ConditionPodsDrained, + nvsentinelv1alpha1.ConditionRemediated, + nvsentinelv1alpha1.ConditionResolved, + } { + SetCondition(&healthEvent.Status.Conditions, nvsentinelv1alpha1.HealthEventCondition{ + Type: condType, + Status: metav1.ConditionUnknown, + LastTransitionTime: now, + Reason: "Pending", + Message: "Waiting for processing", + ObservedGeneration: healthEvent.Generation, + }) + } +} diff --git a/pkg/controllers/healthevents/drain_controller.go b/pkg/controllers/healthevents/drain_controller.go new file mode 100644 index 000000000..96fcdac3b --- /dev/null +++ b/pkg/controllers/healthevents/drain_controller.go @@ -0,0 +1,365 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package healthevents + +import ( + "context" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + + nvsentinelv1alpha1 "github.com/nvidia/nvsentinel/api/nvsentinel/v1alpha1" +) + +// DrainController reconciles HealthEvent resources to drain pods from quarantined nodes. +// It watches for health events in Quarantined phase and evicts pods from the affected node. +type DrainController struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder + + // MaxConcurrentReconciles is the maximum number of concurrent Reconciles. + MaxConcurrentReconciles int + + // GracePeriodSeconds is the default grace period for pod eviction. + // If not set, defaults to 30 seconds. + GracePeriodSeconds int64 + + // DeleteEmptyDirData allows eviction of pods with emptyDir volumes. + DeleteEmptyDirData bool + + // IgnoreDaemonSets skips DaemonSet-managed pods during drain. + IgnoreDaemonSets bool +} + +// DrainResult tracks the outcome of a drain operation. +type DrainResult struct { + PodsEvicted int + PodsFailed int + PodsSkipped int + ErrorMessage string +} + +// +kubebuilder:rbac:groups=nvsentinel.nvidia.com,resources=healthevents,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=nvsentinel.nvidia.com,resources=healthevents/status,verbs=get;update;patch +// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;delete +// +kubebuilder:rbac:groups="",resources=pods/eviction,verbs=create +// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch + +// Reconcile handles HealthEvent reconciliation for draining. +// It processes events in phase=Quarantined, evicts pods from the node, +// and transitions the event to phase=Drained. +func (r *DrainController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := klog.FromContext(ctx).WithValues("healthevent", req.Name) + log.V(1).Info("Reconciling HealthEvent for drain") + + // Fetch the HealthEvent + var healthEvent nvsentinelv1alpha1.HealthEvent + if err := r.Get(ctx, req.NamespacedName, &healthEvent); err != nil { + if apierrors.IsNotFound(err) { + log.V(1).Info("HealthEvent not found, likely deleted") + return ctrl.Result{}, nil + } + log.Error(err, "Failed to get HealthEvent") + return ctrl.Result{}, err + } + + // Skip if not in Quarantined or Draining phase + if healthEvent.Status.Phase != nvsentinelv1alpha1.PhaseQuarantined && + healthEvent.Status.Phase != nvsentinelv1alpha1.PhaseDraining { + log.V(2).Info("Skipping HealthEvent not in Quarantined or Draining phase", "phase", healthEvent.Status.Phase) + return ctrl.Result{}, nil + } + + // Check if drain is skipped via overrides + if healthEvent.Spec.Overrides != nil && + healthEvent.Spec.Overrides.Drain != nil && + healthEvent.Spec.Overrides.Drain.Skip { + log.Info("Drain skipped via override", "nodeName", healthEvent.Spec.NodeName) + return r.transitionToDrained(ctx, &healthEvent, "DrainSkipped", "Drain skipped via override", DrainResult{}) + } + + // Get pods on the node + nodeName := healthEvent.Spec.NodeName + pods, err := r.getPodsOnNode(ctx, nodeName) + if err != nil { + log.Error(err, "Failed to list pods on node", "nodeName", nodeName) + return ctrl.Result{RequeueAfter: 10 * time.Second}, err + } + + // Filter pods to evict + podsToEvict := r.filterPodsForEviction(pods) + log.Info("Draining node", "nodeName", nodeName, "totalPods", len(pods), "podsToEvict", len(podsToEvict)) + + // Update phase to Draining + if err := r.updatePhaseToDraining(ctx, &healthEvent); err != nil { + log.Error(err, "Failed to update phase to Draining") + return ctrl.Result{}, err + } + + // Evict pods + result := r.evictPods(ctx, podsToEvict, nodeName) + + // Record events + if result.PodsEvicted > 0 { + r.Recorder.Eventf(&healthEvent, corev1.EventTypeNormal, "PodsEvicted", + "Evicted %d pods from node %s", result.PodsEvicted, nodeName) + } + if result.PodsFailed > 0 { + r.Recorder.Eventf(&healthEvent, corev1.EventTypeWarning, "EvictionFailed", + "Failed to evict %d pods from node %s: %s", result.PodsFailed, nodeName, result.ErrorMessage) + } + + // If there are failures, requeue to retry + if result.PodsFailed > 0 { + log.Info("Some pods failed to evict, will retry", + "podsEvicted", result.PodsEvicted, + "podsFailed", result.PodsFailed, + ) + return ctrl.Result{RequeueAfter: 15 * time.Second}, nil + } + + // All pods evicted, transition to Drained + return r.transitionToDrained(ctx, &healthEvent, "DrainComplete", + fmt.Sprintf("Successfully evicted %d pods from node %s", result.PodsEvicted, nodeName), + result) +} + +// getPodsOnNode lists all pods running on the specified node. +func (r *DrainController) getPodsOnNode(ctx context.Context, nodeName string) ([]corev1.Pod, error) { + var podList corev1.PodList + if err := r.List(ctx, &podList, &client.ListOptions{ + FieldSelector: fields.SelectorFromSet(fields.Set{"spec.nodeName": nodeName}), + }); err != nil { + return nil, fmt.Errorf("failed to list pods: %w", err) + } + return podList.Items, nil +} + +// filterPodsForEviction filters pods that should be evicted. +// It excludes mirror pods, completed pods, and optionally DaemonSet pods. +func (r *DrainController) filterPodsForEviction(pods []corev1.Pod) []corev1.Pod { + var filtered []corev1.Pod + + for i := range pods { + pod := &pods[i] + + // Skip mirror pods (static pods) + if _, ok := pod.Annotations[corev1.MirrorPodAnnotationKey]; ok { + continue + } + + // Skip completed pods + if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { + continue + } + + // Skip DaemonSet pods if configured + if r.IgnoreDaemonSets && isDaemonSetPod(pod) { + continue + } + + // Skip pods with local storage if not allowed + if !r.DeleteEmptyDirData && hasLocalStorage(pod) { + continue + } + + filtered = append(filtered, pods[i]) + } + + return filtered +} + +// isDaemonSetPod checks if a pod is managed by a DaemonSet. +func isDaemonSetPod(pod *corev1.Pod) bool { + for _, owner := range pod.OwnerReferences { + if owner.Kind == "DaemonSet" { + return true + } + } + return false +} + +// hasLocalStorage checks if a pod uses emptyDir volumes. +func hasLocalStorage(pod *corev1.Pod) bool { + for _, vol := range pod.Spec.Volumes { + if vol.EmptyDir != nil { + return true + } + } + return false +} + +// evictPods evicts the specified pods. +func (r *DrainController) evictPods(ctx context.Context, pods []corev1.Pod, nodeName string) DrainResult { + log := klog.FromContext(ctx) + result := DrainResult{} + + gracePeriod := r.GracePeriodSeconds + if gracePeriod == 0 { + gracePeriod = 30 + } + + for i := range pods { + pod := &pods[i] + + eviction := &policyv1.Eviction{ + ObjectMeta: metav1.ObjectMeta{ + Name: pod.Name, + Namespace: pod.Namespace, + }, + DeleteOptions: &metav1.DeleteOptions{ + GracePeriodSeconds: &gracePeriod, + }, + } + + err := r.SubResource("eviction").Create(ctx, pod, eviction) + if err != nil { + if apierrors.IsNotFound(err) { + // Pod already deleted + result.PodsEvicted++ + continue + } + if apierrors.IsTooManyRequests(err) { + // PDB violation - record but don't count as failure + log.V(1).Info("Eviction blocked by PDB, will retry", + "pod", pod.Name, + "namespace", pod.Namespace, + ) + result.PodsFailed++ + result.ErrorMessage = "blocked by PodDisruptionBudget" + continue + } + + log.Error(err, "Failed to evict pod", + "pod", pod.Name, + "namespace", pod.Namespace, + ) + result.PodsFailed++ + result.ErrorMessage = err.Error() + continue + } + + log.V(1).Info("Evicted pod", + "pod", pod.Name, + "namespace", pod.Namespace, + "nodeName", nodeName, + ) + result.PodsEvicted++ + + // Record metric + drainActionsTotal.WithLabelValues(nodeName, "evicted").Inc() + } + + return result +} + +// updatePhaseToDraining updates the HealthEvent to Draining phase. +func (r *DrainController) updatePhaseToDraining(ctx context.Context, healthEvent *nvsentinelv1alpha1.HealthEvent) error { + // Only update if still in Quarantined phase + if healthEvent.Status.Phase != nvsentinelv1alpha1.PhaseQuarantined { + return nil + } + + patch := client.MergeFrom(healthEvent.DeepCopy()) + healthEvent.Status.Phase = nvsentinelv1alpha1.PhaseDraining + healthEvent.Status.ObservedGeneration = healthEvent.Generation + + return r.Status().Patch(ctx, healthEvent, patch) +} + +// transitionToDrained updates the HealthEvent to Drained phase. +func (r *DrainController) transitionToDrained( + ctx context.Context, + healthEvent *nvsentinelv1alpha1.HealthEvent, + reason, message string, + result DrainResult, +) (ctrl.Result, error) { + log := klog.FromContext(ctx) + + patch := client.MergeFrom(healthEvent.DeepCopy()) + + // Update phase + healthEvent.Status.Phase = nvsentinelv1alpha1.PhaseDrained + healthEvent.Status.ObservedGeneration = healthEvent.Generation + + // Update conditions + now := metav1.Now() + SetCondition(&healthEvent.Status.Conditions, nvsentinelv1alpha1.HealthEventCondition{ + Type: nvsentinelv1alpha1.ConditionPodsDrained, + Status: metav1.ConditionTrue, + LastTransitionTime: now, + Reason: reason, + Message: message, + ObservedGeneration: healthEvent.Generation, + }) + + if err := r.Status().Patch(ctx, healthEvent, patch); err != nil { + log.Error(err, "Failed to update HealthEvent status") + return ctrl.Result{}, err + } + + log.Info("HealthEvent transitioned to Drained", + "healthEvent", healthEvent.Name, + "nodeName", healthEvent.Spec.NodeName, + "podsEvicted", result.PodsEvicted, + "podsSkipped", result.PodsSkipped, + ) + + // Increment metrics + drainActionsTotal.WithLabelValues(healthEvent.Spec.NodeName, "completed").Inc() + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +// Note: We intentionally do NOT use phase-based predicates because status subresource +// updates may not trigger watch events reliably. Instead, we filter by phase in Reconcile. +func (r *DrainController) SetupWithManager(mgr ctrl.Manager) error { + // Register metrics + registerDrainMetrics() + + maxConcurrent := r.MaxConcurrentReconciles + if maxConcurrent == 0 { + maxConcurrent = 2 // Default - lower than quarantine to avoid overwhelming API + } + + // Set defaults for zero-value fields that have non-zero defaults + if r.GracePeriodSeconds == 0 { + r.GracePeriodSeconds = 30 + } + // Note: DeleteEmptyDirData and IgnoreDaemonSets default to false (Go zero value). + // Callers should explicitly set these to true if desired. The controller-test + // and standard deployments set both to true. + + return ctrl.NewControllerManagedBy(mgr). + For(&nvsentinelv1alpha1.HealthEvent{}). + WithOptions(controller.Options{ + MaxConcurrentReconciles: maxConcurrent, + }). + Named("drain"). + Complete(r) +} diff --git a/pkg/controllers/healthevents/drain_controller_test.go b/pkg/controllers/healthevents/drain_controller_test.go new file mode 100644 index 000000000..cfd8b6203 --- /dev/null +++ b/pkg/controllers/healthevents/drain_controller_test.go @@ -0,0 +1,404 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package healthevents + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + nvsentinelv1alpha1 "github.com/nvidia/nvsentinel/api/nvsentinel/v1alpha1" +) + +func TestDrainController_Reconcile(t *testing.T) { + scheme := runtime.NewScheme() + _ = nvsentinelv1alpha1.AddToScheme(scheme) + _ = corev1.AddToScheme(scheme) + + // Note: The fake client doesn't support field selectors by default. + // For real integration tests, use envtest. For unit tests, we test + // the logic paths that don't require field selectors (skip override, wrong phase). + + tests := []struct { + name string + healthEvent *nvsentinelv1alpha1.HealthEvent + pods []corev1.Pod + wantPhase nvsentinelv1alpha1.HealthEventPhase + wantRequeue bool + skipPodList bool // Skip tests that require field selector + }{ + { + name: "quarantined event with no pods should drain", + healthEvent: &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-drain-1", + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "nvml-health-monitor", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "xid-error-check", + IsFatal: true, + DetectedAt: metav1.Now(), + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseQuarantined, + }, + }, + pods: []corev1.Pod{}, + wantPhase: nvsentinelv1alpha1.PhaseQuarantined, // Won't change - fake client doesn't support field selector + wantRequeue: false, + skipPodList: true, // Skip - fake client doesn't support field selectors + }, + { + name: "event not in quarantined phase should be skipped", + healthEvent: &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-drain-2", + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "nvml-health-monitor", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "xid-error-check", + IsFatal: true, + DetectedAt: metav1.Now(), + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseNew, + }, + }, + pods: []corev1.Pod{}, + wantPhase: nvsentinelv1alpha1.PhaseNew, + wantRequeue: false, + skipPodList: false, + }, + { + name: "event with drain skip override should transition to drained", + healthEvent: &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-drain-3", + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "nvml-health-monitor", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "xid-error-check", + IsFatal: true, + DetectedAt: metav1.Now(), + Overrides: &nvsentinelv1alpha1.BehaviourOverrides{ + Drain: &nvsentinelv1alpha1.ActionOverride{ + Skip: true, + }, + }, + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseQuarantined, + }, + }, + pods: []corev1.Pod{}, + wantPhase: nvsentinelv1alpha1.PhaseDrained, + wantRequeue: false, + skipPodList: false, + }, + { + name: "event in Draining phase should continue drain", + healthEvent: &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-draining-reentry", + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + NodeName: "node-1", + Source: "test", + ComponentClass: "GPU", + CheckName: "xid-check", + IsFatal: true, + DetectedAt: metav1.Now(), + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseDraining, + }, + }, + wantPhase: nvsentinelv1alpha1.PhaseDrained, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.skipPodList { + t.Skip("Skipping test that requires field selector (use envtest for integration tests)") + } + + objs := []client.Object{tt.healthEvent} + for i := range tt.pods { + objs = append(objs, &tt.pods[i]) + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objs...). + WithStatusSubresource(&nvsentinelv1alpha1.HealthEvent{}). + WithIndex(&corev1.Pod{}, "spec.nodeName", func(obj client.Object) []string { + pod, ok := obj.(*corev1.Pod) + if !ok { + return nil + } + if pod.Spec.NodeName == "" { + return nil + } + return []string{pod.Spec.NodeName} + }). + Build() + + recorder := record.NewFakeRecorder(10) + + r := &DrainController{ + Client: fakeClient, + Scheme: scheme, + Recorder: recorder, + IgnoreDaemonSets: true, + DeleteEmptyDirData: true, + } + + ctx := context.Background() + req := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: tt.healthEvent.Name, + }, + } + + result, err := r.Reconcile(ctx, req) + if err != nil { + t.Errorf("Reconcile() error = %v", err) + return + } + + if result.Requeue != tt.wantRequeue { + t.Errorf("Reconcile() requeue = %v, want %v", result.Requeue, tt.wantRequeue) + } + + // Check HealthEvent phase + var updatedEvent nvsentinelv1alpha1.HealthEvent + if err := fakeClient.Get(ctx, req.NamespacedName, &updatedEvent); err != nil { + t.Errorf("Failed to get updated HealthEvent: %v", err) + return + } + + if updatedEvent.Status.Phase != tt.wantPhase { + t.Errorf("HealthEvent phase = %v, want %v", updatedEvent.Status.Phase, tt.wantPhase) + } + }) + } +} + +func TestDrainController_FilterPodsForEviction(t *testing.T) { + r := &DrainController{ + IgnoreDaemonSets: true, + DeleteEmptyDirData: false, + } + + tests := []struct { + name string + pods []corev1.Pod + wantLen int + }{ + { + name: "filter out mirror pods", + pods: []corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "regular-pod", + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "mirror-pod", + Annotations: map[string]string{ + corev1.MirrorPodAnnotationKey: "true", + }, + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + }, + }, + wantLen: 1, + }, + { + name: "filter out completed pods", + pods: []corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{Name: "running-pod"}, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "succeeded-pod"}, + Status: corev1.PodStatus{Phase: corev1.PodSucceeded}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "failed-pod"}, + Status: corev1.PodStatus{Phase: corev1.PodFailed}, + }, + }, + wantLen: 1, + }, + { + name: "filter out daemonset pods when configured", + pods: []corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{Name: "regular-pod"}, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "daemonset-pod", + OwnerReferences: []metav1.OwnerReference{ + {Kind: "DaemonSet", Name: "my-ds"}, + }, + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + }, + }, + wantLen: 1, + }, + { + name: "filter out pods with emptyDir when not allowed", + pods: []corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{Name: "regular-pod"}, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "emptydir-pod"}, + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{ + {Name: "data", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + }, + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + }, + }, + wantLen: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + filtered := r.filterPodsForEviction(tt.pods) + if len(filtered) != tt.wantLen { + t.Errorf("filterPodsForEviction() returned %d pods, want %d", len(filtered), tt.wantLen) + } + }) + } +} + +func TestIsDaemonSetPod(t *testing.T) { + tests := []struct { + name string + pod *corev1.Pod + want bool + }{ + { + name: "pod with daemonset owner", + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + OwnerReferences: []metav1.OwnerReference{ + {Kind: "DaemonSet", Name: "my-ds"}, + }, + }, + }, + want: true, + }, + { + name: "pod with deployment owner", + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + OwnerReferences: []metav1.OwnerReference{ + {Kind: "ReplicaSet", Name: "my-rs"}, + }, + }, + }, + want: false, + }, + { + name: "pod with no owner", + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{}, + }, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isDaemonSetPod(tt.pod); got != tt.want { + t.Errorf("isDaemonSetPod() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestHasLocalStorage(t *testing.T) { + tests := []struct { + name string + pod *corev1.Pod + want bool + }{ + { + name: "pod with emptyDir", + pod: &corev1.Pod{ + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{ + {Name: "data", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + }, + }, + }, + want: true, + }, + { + name: "pod with configMap volume", + pod: &corev1.Pod{ + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{ + {Name: "config", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{}}}, + }, + }, + }, + want: false, + }, + { + name: "pod with no volumes", + pod: &corev1.Pod{ + Spec: corev1.PodSpec{}, + }, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := hasLocalStorage(tt.pod); got != tt.want { + t.Errorf("hasLocalStorage() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/controllers/healthevents/metrics.go b/pkg/controllers/healthevents/metrics.go new file mode 100644 index 000000000..7fc499195 --- /dev/null +++ b/pkg/controllers/healthevents/metrics.go @@ -0,0 +1,142 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package healthevents + +import ( + "sync" + + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +var ( + registerOnce sync.Once + + // quarantineActionsTotal tracks the number of quarantine actions taken. + quarantineActionsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "nvsentinel", + Subsystem: "quarantine_controller", + Name: "actions_total", + Help: "Total number of quarantine actions taken by outcome", + }, + []string{"node", "outcome"}, // outcome: success, failed, skipped + ) +) + +// registerMetrics registers all metrics with the controller-runtime metrics registry. +func registerMetrics() { + registerOnce.Do(func() { + metrics.Registry.MustRegister( + quarantineActionsTotal, + ) + }) +} + +// ============================================================================= +// Drain Controller Metrics +// ============================================================================= + +var ( + registerDrainOnce sync.Once + + // drainActionsTotal tracks the number of drain actions taken. + drainActionsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "nvsentinel", + Subsystem: "drain_controller", + Name: "actions_total", + Help: "Total number of drain actions taken by outcome", + }, + []string{"node", "outcome"}, // outcome: evicted, failed, skipped, completed + ) +) + +// registerDrainMetrics registers drain controller metrics. +func registerDrainMetrics() { + registerDrainOnce.Do(func() { + metrics.Registry.MustRegister( + drainActionsTotal, + ) + }) +} + +// ============================================================================= +// TTL Controller Metrics +// ============================================================================= + +var ( + registerTTLOnce sync.Once + + // ttlDeletionsTotal tracks the number of events deleted by TTL. + ttlDeletionsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "nvsentinel", + Subsystem: "ttl_controller", + Name: "deletions_total", + Help: "Total number of HealthEvents deleted by TTL controller", + }, + []string{"node", "phase"}, + ) +) + +// registerTTLMetrics registers TTL controller metrics. +func registerTTLMetrics() { + registerTTLOnce.Do(func() { + metrics.Registry.MustRegister( + ttlDeletionsTotal, + ) + }) +} + +// ============================================================================= +// Remediation Controller Metrics +// ============================================================================= + +var ( + registerRemediationOnce sync.Once + + // remediationActionsTotal tracks successful remediation actions. + remediationActionsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "nvsentinel", + Subsystem: "remediation_controller", + Name: "actions_total", + Help: "Total number of remediation actions executed", + }, + []string{"node", "strategy"}, + ) + + // remediationFailuresTotal tracks failed remediation attempts. + remediationFailuresTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "nvsentinel", + Subsystem: "remediation_controller", + Name: "failures_total", + Help: "Total number of failed remediation attempts", + }, + []string{"node", "strategy"}, + ) +) + +// registerRemediationMetrics registers remediation controller metrics. +func registerRemediationMetrics() { + registerRemediationOnce.Do(func() { + metrics.Registry.MustRegister( + remediationActionsTotal, + remediationFailuresTotal, + ) + }) +} diff --git a/pkg/controllers/healthevents/quarantine_controller.go b/pkg/controllers/healthevents/quarantine_controller.go new file mode 100644 index 000000000..d083ee21d --- /dev/null +++ b/pkg/controllers/healthevents/quarantine_controller.go @@ -0,0 +1,293 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package healthevents + +import ( + "context" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + + nvsentinelv1alpha1 "github.com/nvidia/nvsentinel/api/nvsentinel/v1alpha1" +) + +// QuarantineController reconciles HealthEvent resources to quarantine nodes. +// It watches for new fatal health events (phase=New, isFatal=true) and +// cordons the affected node to prevent new workloads from being scheduled. +type QuarantineController struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder + + // MaxConcurrentReconciles is the maximum number of concurrent Reconciles. + MaxConcurrentReconciles int +} + +// +kubebuilder:rbac:groups=nvsentinel.nvidia.com,resources=healthevents,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=nvsentinel.nvidia.com,resources=healthevents/status,verbs=get;update;patch +// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch + +// Reconcile handles HealthEvent reconciliation. +// It processes events in phase=New with isFatal=true, cordons the node, +// and transitions the event to phase=Quarantined. +func (r *QuarantineController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := klog.FromContext(ctx).WithValues("healthevent", req.Name) + log.V(1).Info("Reconciling HealthEvent") + + // Fetch the HealthEvent + var healthEvent nvsentinelv1alpha1.HealthEvent + if err := r.Get(ctx, req.NamespacedName, &healthEvent); err != nil { + if apierrors.IsNotFound(err) { + log.V(1).Info("HealthEvent not found, likely deleted") + return ctrl.Result{}, nil + } + log.Error(err, "Failed to get HealthEvent") + return ctrl.Result{}, err + } + + // Skip if already processed or not in New phase + // Note: Empty phase ("") is treated as New because status is a subresource + // and isn't persisted during resource creation + if healthEvent.Status.Phase != nvsentinelv1alpha1.PhaseNew && healthEvent.Status.Phase != "" { + log.V(2).Info("Skipping HealthEvent not in New phase", "phase", healthEvent.Status.Phase) + return ctrl.Result{}, nil + } + + // Skip non-fatal events + if !healthEvent.Spec.IsFatal { + log.V(2).Info("Skipping non-fatal HealthEvent") + // Mark as resolved since we won't process it further + return r.markAsResolved(ctx, &healthEvent, "NonFatal", "Event is not fatal, no quarantine needed") + } + + // Check if quarantine is skipped via overrides + if healthEvent.Spec.Overrides != nil && + healthEvent.Spec.Overrides.Quarantine != nil && + healthEvent.Spec.Overrides.Quarantine.Skip { + log.Info("Quarantine skipped via override", "nodeName", healthEvent.Spec.NodeName) + return r.markAsCancelled(ctx, &healthEvent, "QuarantineSkipped", "Quarantine skipped via override") + } + + // Get the node + nodeName := healthEvent.Spec.NodeName + var node corev1.Node + if err := r.Get(ctx, client.ObjectKey{Name: nodeName}, &node); err != nil { + if apierrors.IsNotFound(err) { + log.Info("Node not found, cancelling health event", "nodeName", nodeName) + r.Recorder.Eventf(&healthEvent, corev1.EventTypeWarning, "NodeNotFound", + "Node %s not found, cancelling event", nodeName) + return r.markAsCancelled(ctx, &healthEvent, "NodeNotFound", + fmt.Sprintf("Node %s not found", nodeName)) + } + log.Error(err, "Failed to get node", "nodeName", nodeName) + return ctrl.Result{}, err + } + + // Check if node is already cordoned + if node.Spec.Unschedulable { + log.Info("Node already cordoned", "nodeName", nodeName) + // Node is already cordoned, transition to Quarantined + return r.transitionToQuarantined(ctx, &healthEvent, "NodeAlreadyCordoned", + "Node was already cordoned") + } + + // Cordon the node + log.Info("Cordoning node", "nodeName", nodeName) + if err := r.cordonNode(ctx, &node); err != nil { + log.Error(err, "Failed to cordon node", "nodeName", nodeName) + r.Recorder.Eventf(&healthEvent, corev1.EventTypeWarning, "CordonFailed", + "Failed to cordon node %s: %v", nodeName, err) + // Retry after backoff + return ctrl.Result{RequeueAfter: 5 * time.Second}, err + } + + // Record event + r.Recorder.Eventf(&healthEvent, corev1.EventTypeNormal, "NodeCordoned", + "Node %s cordoned due to fatal GPU error", nodeName) + r.Recorder.Eventf(&node, corev1.EventTypeWarning, "Quarantined", + "Node cordoned by NVSentinel due to HealthEvent %s", healthEvent.Name) + + // Transition to Quarantined + return r.transitionToQuarantined(ctx, &healthEvent, "FatalError", + fmt.Sprintf("Node cordoned due to fatal %s error", healthEvent.Spec.ComponentClass)) +} + +// cordonNode sets the node's Unschedulable field to true. +func (r *QuarantineController) cordonNode(ctx context.Context, node *corev1.Node) error { + patch := client.MergeFrom(node.DeepCopy()) + node.Spec.Unschedulable = true + + // Add taint for immediate effect on pending pods + quarantineTaint := corev1.Taint{ + Key: "nvsentinel.nvidia.com/quarantine", + Value: "true", + Effect: corev1.TaintEffectNoSchedule, + } + + // Check if taint already exists + hasTaint := false + for _, t := range node.Spec.Taints { + if t.Key == quarantineTaint.Key { + hasTaint = true + break + } + } + if !hasTaint { + node.Spec.Taints = append(node.Spec.Taints, quarantineTaint) + } + + return r.Patch(ctx, node, patch) +} + +// transitionToQuarantined updates the HealthEvent to Quarantined phase. +func (r *QuarantineController) transitionToQuarantined( + ctx context.Context, + healthEvent *nvsentinelv1alpha1.HealthEvent, + reason, message string, +) (ctrl.Result, error) { + log := klog.FromContext(ctx) + + patch := client.MergeFrom(healthEvent.DeepCopy()) + + // Update phase + healthEvent.Status.Phase = nvsentinelv1alpha1.PhaseQuarantined + healthEvent.Status.ObservedGeneration = healthEvent.Generation + + // Update conditions + now := metav1.Now() + SetCondition(&healthEvent.Status.Conditions, nvsentinelv1alpha1.HealthEventCondition{ + Type: nvsentinelv1alpha1.ConditionNodeQuarantined, + Status: metav1.ConditionTrue, + LastTransitionTime: now, + Reason: reason, + Message: message, + ObservedGeneration: healthEvent.Generation, + }) + + if err := r.Status().Patch(ctx, healthEvent, patch); err != nil { + log.Error(err, "Failed to update HealthEvent status") + return ctrl.Result{}, err + } + + log.Info("HealthEvent transitioned to Quarantined", + "healthEvent", healthEvent.Name, + "nodeName", healthEvent.Spec.NodeName) + + // Increment metrics + quarantineActionsTotal.WithLabelValues(healthEvent.Spec.NodeName, "success").Inc() + + return ctrl.Result{}, nil +} + +// markAsResolved updates the HealthEvent to Resolved phase for non-fatal events. +func (r *QuarantineController) markAsResolved( + ctx context.Context, + healthEvent *nvsentinelv1alpha1.HealthEvent, + reason, message string, +) (ctrl.Result, error) { + log := klog.FromContext(ctx) + + patch := client.MergeFrom(healthEvent.DeepCopy()) + + healthEvent.Status.Phase = nvsentinelv1alpha1.PhaseResolved + healthEvent.Status.ObservedGeneration = healthEvent.Generation + + now := metav1.Now() + healthEvent.Status.ResolvedAt = &now + + SetCondition(&healthEvent.Status.Conditions, nvsentinelv1alpha1.HealthEventCondition{ + Type: nvsentinelv1alpha1.ConditionResolved, + Status: metav1.ConditionTrue, + LastTransitionTime: now, + Reason: reason, + Message: message, + ObservedGeneration: healthEvent.Generation, + }) + + if err := r.Status().Patch(ctx, healthEvent, patch); err != nil { + log.Error(err, "Failed to update HealthEvent status") + return ctrl.Result{}, err + } + + log.V(1).Info("HealthEvent marked as Resolved", "healthEvent", healthEvent.Name) + return ctrl.Result{}, nil +} + +// markAsCancelled updates the HealthEvent to Cancelled phase. +func (r *QuarantineController) markAsCancelled( + ctx context.Context, + healthEvent *nvsentinelv1alpha1.HealthEvent, + reason, message string, +) (ctrl.Result, error) { + log := klog.FromContext(ctx) + + patch := client.MergeFrom(healthEvent.DeepCopy()) + + healthEvent.Status.Phase = nvsentinelv1alpha1.PhaseCancelled + healthEvent.Status.ObservedGeneration = healthEvent.Generation + + now := metav1.Now() + healthEvent.Status.ResolvedAt = &now + + SetCondition(&healthEvent.Status.Conditions, nvsentinelv1alpha1.HealthEventCondition{ + Type: nvsentinelv1alpha1.ConditionResolved, + Status: metav1.ConditionTrue, + LastTransitionTime: now, + Reason: reason, + Message: message, + ObservedGeneration: healthEvent.Generation, + }) + + if err := r.Status().Patch(ctx, healthEvent, patch); err != nil { + log.Error(err, "Failed to update HealthEvent status") + return ctrl.Result{}, err + } + + log.Info("HealthEvent cancelled", "healthEvent", healthEvent.Name, "reason", reason) + quarantineActionsTotal.WithLabelValues(healthEvent.Spec.NodeName, "skipped").Inc() + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +// Note: We intentionally do NOT use phase-based predicates because status subresource +// updates may not trigger watch events reliably. Instead, we filter by phase in Reconcile. +func (r *QuarantineController) SetupWithManager(mgr ctrl.Manager) error { + // Initialize metrics + registerMetrics() + + maxConcurrent := r.MaxConcurrentReconciles + if maxConcurrent == 0 { + maxConcurrent = 3 // Default + } + + return ctrl.NewControllerManagedBy(mgr). + For(&nvsentinelv1alpha1.HealthEvent{}). + WithOptions(controller.Options{ + MaxConcurrentReconciles: maxConcurrent, + }). + Named("quarantine"). + Complete(r) +} diff --git a/pkg/controllers/healthevents/quarantine_controller_test.go b/pkg/controllers/healthevents/quarantine_controller_test.go new file mode 100644 index 000000000..d3ab111a2 --- /dev/null +++ b/pkg/controllers/healthevents/quarantine_controller_test.go @@ -0,0 +1,430 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package healthevents + +import ( + "context" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + nvsentinelv1alpha1 "github.com/nvidia/nvsentinel/api/nvsentinel/v1alpha1" +) + +func TestQuarantineController_Reconcile(t *testing.T) { + scheme := runtime.NewScheme() + _ = nvsentinelv1alpha1.AddToScheme(scheme) + _ = corev1.AddToScheme(scheme) + + tests := []struct { + name string + healthEvent *nvsentinelv1alpha1.HealthEvent + node *corev1.Node + wantPhase nvsentinelv1alpha1.HealthEventPhase + wantNodeCordoned bool + wantRequeue bool + }{ + { + name: "fatal event should quarantine node", + healthEvent: &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-event-1", + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "nvml-health-monitor", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "xid-error-check", + IsFatal: true, + IsHealthy: false, + DetectedAt: metav1.Now(), + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseNew, + }, + }, + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-1", + }, + Spec: corev1.NodeSpec{ + Unschedulable: false, + }, + }, + wantPhase: nvsentinelv1alpha1.PhaseQuarantined, + wantNodeCordoned: true, + wantRequeue: false, + }, + { + name: "non-fatal event should be resolved", + healthEvent: &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-event-2", + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "nvml-health-monitor", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "memory-ecc-check", + IsFatal: false, + IsHealthy: false, + DetectedAt: metav1.Now(), + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseNew, + }, + }, + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-1", + }, + }, + wantPhase: nvsentinelv1alpha1.PhaseResolved, + wantNodeCordoned: false, + wantRequeue: false, + }, + { + name: "already quarantined phase should be skipped", + healthEvent: &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-event-3", + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "nvml-health-monitor", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "xid-error-check", + IsFatal: true, + IsHealthy: false, + DetectedAt: metav1.Now(), + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseQuarantined, + }, + }, + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-1", + }, + Spec: corev1.NodeSpec{ + Unschedulable: true, + }, + }, + wantPhase: nvsentinelv1alpha1.PhaseQuarantined, + wantNodeCordoned: true, + wantRequeue: false, + }, + { + name: "event with skip override should be cancelled", + healthEvent: &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-event-4", + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "nvml-health-monitor", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "xid-error-check", + IsFatal: true, + IsHealthy: false, + DetectedAt: metav1.Now(), + Overrides: &nvsentinelv1alpha1.BehaviourOverrides{ + Quarantine: &nvsentinelv1alpha1.ActionOverride{ + Skip: true, + }, + }, + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseNew, + }, + }, + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-1", + }, + }, + wantPhase: nvsentinelv1alpha1.PhaseCancelled, + wantNodeCordoned: false, + wantRequeue: false, + }, + { + name: "already cordoned node should transition to quarantined", + healthEvent: &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-event-5", + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "nvml-health-monitor", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "xid-error-check", + IsFatal: true, + IsHealthy: false, + DetectedAt: metav1.Now(), + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseNew, + }, + }, + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-1", + }, + Spec: corev1.NodeSpec{ + Unschedulable: true, // Already cordoned + }, + }, + wantPhase: nvsentinelv1alpha1.PhaseQuarantined, + wantNodeCordoned: true, + wantRequeue: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create fake client with objects + objs := []client.Object{tt.healthEvent} + if tt.node != nil { + objs = append(objs, tt.node) + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objs...). + WithStatusSubresource(&nvsentinelv1alpha1.HealthEvent{}). + Build() + + recorder := record.NewFakeRecorder(10) + + r := &QuarantineController{ + Client: fakeClient, + Scheme: scheme, + Recorder: recorder, + } + + // Reconcile + ctx := context.Background() + req := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: tt.healthEvent.Name, + }, + } + + result, err := r.Reconcile(ctx, req) + if err != nil { + t.Errorf("Reconcile() error = %v", err) + return + } + + if result.Requeue != tt.wantRequeue { + t.Errorf("Reconcile() requeue = %v, want %v", result.Requeue, tt.wantRequeue) + } + + // Check HealthEvent phase + var updatedEvent nvsentinelv1alpha1.HealthEvent + if err := fakeClient.Get(ctx, req.NamespacedName, &updatedEvent); err != nil { + t.Errorf("Failed to get updated HealthEvent: %v", err) + return + } + + if updatedEvent.Status.Phase != tt.wantPhase { + t.Errorf("HealthEvent phase = %v, want %v", updatedEvent.Status.Phase, tt.wantPhase) + } + + // Check node cordon status + if tt.node != nil { + var updatedNode corev1.Node + if err := fakeClient.Get(ctx, types.NamespacedName{Name: tt.node.Name}, &updatedNode); err != nil { + t.Errorf("Failed to get updated Node: %v", err) + return + } + + if updatedNode.Spec.Unschedulable != tt.wantNodeCordoned { + t.Errorf("Node.Spec.Unschedulable = %v, want %v", + updatedNode.Spec.Unschedulable, tt.wantNodeCordoned) + } + } + }) + } +} + +func TestQuarantineController_Reconcile_NodeNotFound(t *testing.T) { + scheme := runtime.NewScheme() + _ = nvsentinelv1alpha1.AddToScheme(scheme) + _ = corev1.AddToScheme(scheme) + + healthEvent := &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-event-missing-node", + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "nvml-health-monitor", + NodeName: "nonexistent-node", + ComponentClass: "GPU", + CheckName: "xid-error-check", + IsFatal: true, + IsHealthy: false, + DetectedAt: metav1.Now(), + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseNew, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(healthEvent). + WithStatusSubresource(&nvsentinelv1alpha1.HealthEvent{}). + Build() + + recorder := record.NewFakeRecorder(10) + + r := &QuarantineController{ + Client: fakeClient, + Scheme: scheme, + Recorder: recorder, + } + + ctx := context.Background() + req := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: healthEvent.Name, + }, + } + + // Should not error - just records warning event and cancels the event + result, err := r.Reconcile(ctx, req) + if err != nil { + t.Errorf("Reconcile() error = %v, want nil", err) + } + if result.Requeue { + t.Errorf("Reconcile() requeue = %v, want false", result.Requeue) + } + + // Check that a warning event was recorded + select { + case event := <-recorder.Events: + if event == "" { + t.Error("Expected warning event to be recorded") + } + case <-time.After(time.Second): + t.Error("Timeout waiting for event") + } + + // The HealthEvent should transition to Cancelled phase + var updatedEvent nvsentinelv1alpha1.HealthEvent + if err := fakeClient.Get(ctx, req.NamespacedName, &updatedEvent); err != nil { + t.Fatalf("Failed to get updated HealthEvent: %v", err) + } + if updatedEvent.Status.Phase != nvsentinelv1alpha1.PhaseCancelled { + t.Errorf("HealthEvent phase = %v, want %v", updatedEvent.Status.Phase, nvsentinelv1alpha1.PhaseCancelled) + } +} + +func TestConditionHelpers(t *testing.T) { + now := metav1.Now() + + t.Run("SetCondition adds new condition", func(t *testing.T) { + var conditions []nvsentinelv1alpha1.HealthEventCondition + + SetCondition(&conditions, nvsentinelv1alpha1.HealthEventCondition{ + Type: nvsentinelv1alpha1.ConditionDetected, + Status: metav1.ConditionTrue, + LastTransitionTime: now, + Reason: "Test", + }) + + if len(conditions) != 1 { + t.Errorf("Expected 1 condition, got %d", len(conditions)) + } + if conditions[0].Type != nvsentinelv1alpha1.ConditionDetected { + t.Errorf("Expected Detected condition, got %s", conditions[0].Type) + } + }) + + t.Run("SetCondition updates existing condition", func(t *testing.T) { + conditions := []nvsentinelv1alpha1.HealthEventCondition{ + { + Type: nvsentinelv1alpha1.ConditionDetected, + Status: metav1.ConditionFalse, + Reason: "Old", + }, + } + + SetCondition(&conditions, nvsentinelv1alpha1.HealthEventCondition{ + Type: nvsentinelv1alpha1.ConditionDetected, + Status: metav1.ConditionTrue, + LastTransitionTime: now, + Reason: "New", + }) + + if len(conditions) != 1 { + t.Errorf("Expected 1 condition, got %d", len(conditions)) + } + if conditions[0].Status != metav1.ConditionTrue { + t.Errorf("Expected True status, got %s", conditions[0].Status) + } + if conditions[0].Reason != "New" { + t.Errorf("Expected New reason, got %s", conditions[0].Reason) + } + }) + + t.Run("GetCondition returns nil for missing", func(t *testing.T) { + conditions := []nvsentinelv1alpha1.HealthEventCondition{} + c := GetCondition(conditions, nvsentinelv1alpha1.ConditionDetected) + if c != nil { + t.Error("Expected nil for missing condition") + } + }) + + t.Run("IsConditionTrue", func(t *testing.T) { + conditions := []nvsentinelv1alpha1.HealthEventCondition{ + { + Type: nvsentinelv1alpha1.ConditionDetected, + Status: metav1.ConditionTrue, + }, + } + if !IsConditionTrue(conditions, nvsentinelv1alpha1.ConditionDetected) { + t.Error("Expected IsConditionTrue to return true") + } + if IsConditionTrue(conditions, nvsentinelv1alpha1.ConditionRemediated) { + t.Error("Expected IsConditionTrue to return false for missing condition") + } + }) + + t.Run("RemoveCondition", func(t *testing.T) { + conditions := []nvsentinelv1alpha1.HealthEventCondition{ + {Type: nvsentinelv1alpha1.ConditionDetected, Status: metav1.ConditionTrue}, + {Type: nvsentinelv1alpha1.ConditionRemediated, Status: metav1.ConditionFalse}, + } + + RemoveCondition(&conditions, nvsentinelv1alpha1.ConditionDetected) + + if len(conditions) != 1 { + t.Errorf("Expected 1 condition after removal, got %d", len(conditions)) + } + if conditions[0].Type != nvsentinelv1alpha1.ConditionRemediated { + t.Errorf("Expected Remediated condition to remain, got %s", conditions[0].Type) + } + }) +} diff --git a/pkg/controllers/healthevents/remediation_controller.go b/pkg/controllers/healthevents/remediation_controller.go new file mode 100644 index 000000000..937623946 --- /dev/null +++ b/pkg/controllers/healthevents/remediation_controller.go @@ -0,0 +1,468 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package healthevents + +import ( + "context" + "errors" + "fmt" + "time" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + + nvsentinelv1alpha1 "github.com/nvidia/nvsentinel/api/nvsentinel/v1alpha1" +) + +// RemediationStrategy defines how to remediate a health event. +type RemediationStrategy string + +const ( + // StrategyReboot reboots the node via a privileged Job. + StrategyReboot RemediationStrategy = "reboot" + + // StrategyTerminate terminates the node (cloud provider replaces it). + StrategyTerminate RemediationStrategy = "terminate" + + // StrategyGPUReset resets the GPU via NVML (for non-fatal issues). + StrategyGPUReset RemediationStrategy = "gpu-reset" + + // StrategyManual requires manual intervention. + StrategyManual RemediationStrategy = "manual" + + // StrategyNone skips remediation entirely. + StrategyNone RemediationStrategy = "none" +) + +// errRebootInProgress indicates a reboot job exists and is still running. +// The caller should requeue to check again later. +var errRebootInProgress = fmt.Errorf("reboot job in progress") + +// RemediationController reconciles HealthEvent resources in the Drained phase. +// It determines the appropriate remediation action and executes it. +type RemediationController struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder + + // DefaultStrategy is the remediation strategy when none is specified. + DefaultStrategy RemediationStrategy + + // RebootJobNamespace is the namespace where reboot Jobs are created. + RebootJobNamespace string + + // RebootJobImage is the container image for reboot Jobs. + RebootJobImage string + + // MaxConcurrentReconciles is the maximum number of concurrent Reconciles. + MaxConcurrentReconciles int + + // RebootJobTTL is how long to keep completed reboot Jobs. + RebootJobTTL time.Duration +} + +// +kubebuilder:rbac:groups=nvsentinel.nvidia.com,resources=healthevents,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=nvsentinel.nvidia.com,resources=healthevents/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=create;get;list;watch;delete +// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch;delete +// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch + +// Reconcile handles HealthEvent remediation. +// It watches for events in the Drained phase and triggers remediation. +func (r *RemediationController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := klog.FromContext(ctx).WithValues("healthevent", req.Name) + log.V(2).Info("Reconciling HealthEvent for remediation") + + // Fetch the HealthEvent + var healthEvent nvsentinelv1alpha1.HealthEvent + if err := r.Get(ctx, req.NamespacedName, &healthEvent); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + log.Error(err, "Failed to get HealthEvent") + return ctrl.Result{}, err + } + + // Only process events in Drained phase + if healthEvent.Status.Phase != nvsentinelv1alpha1.PhaseDrained { + log.V(2).Info("Skipping event not in Drained phase", "phase", healthEvent.Status.Phase) + return ctrl.Result{}, nil + } + + nodeName := healthEvent.Spec.NodeName + log = log.WithValues("nodeName", nodeName) + + // Determine remediation strategy + strategy := r.determineStrategy(&healthEvent) + log.Info("Determined remediation strategy", "strategy", strategy) + + // Execute remediation based on strategy + var err error + var message string + + switch strategy { + case StrategyReboot: + err = r.executeReboot(ctx, &healthEvent, log) + message = "Node reboot initiated" + case StrategyTerminate: + err = r.executeTerminate(ctx, &healthEvent, log) + message = "Node termination initiated" + case StrategyGPUReset: + err = r.executeGPUReset(ctx, &healthEvent, log) + message = "GPU reset initiated" + case StrategyManual: + message = "Manual remediation required" + log.Info("Manual remediation required", "recommendedAction", healthEvent.Spec.RecommendedAction) + case StrategyNone: + message = "No remediation action taken" + log.Info("Remediation skipped") + default: + err = fmt.Errorf("unknown remediation strategy: %s", strategy) + } + + if err != nil { + if errors.Is(err, errRebootInProgress) { + log.Info("Reboot in progress, requeueing") + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } + + log.Error(err, "Remediation failed") + remediationFailuresTotal.WithLabelValues(nodeName, string(strategy)).Inc() + + // Record event + r.Recorder.Event(&healthEvent, corev1.EventTypeWarning, "RemediationFailed", + fmt.Sprintf("Remediation failed: %v", err)) + + // Set condition + patch := client.MergeFrom(healthEvent.DeepCopy()) + now := metav1.Now() + SetCondition(&healthEvent.Status.Conditions, nvsentinelv1alpha1.HealthEventCondition{ + Type: nvsentinelv1alpha1.ConditionRemediated, + Status: metav1.ConditionFalse, + LastTransitionTime: now, + Reason: "RemediationFailed", + Message: err.Error(), + ObservedGeneration: healthEvent.Generation, + }) + + // Patch status with error condition but don't change phase + if updateErr := r.Status().Patch(ctx, &healthEvent, patch); updateErr != nil { + log.Error(updateErr, "Failed to update HealthEvent status") + return ctrl.Result{}, updateErr + } + + // Requeue to retry + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } + + // Remediation succeeded - transition to Remediated phase + if err := r.transitionToRemediated(ctx, &healthEvent, message, log); err != nil { + return ctrl.Result{}, err + } + + // Record metrics + remediationActionsTotal.WithLabelValues(nodeName, string(strategy)).Inc() + + return ctrl.Result{}, nil +} + +// determineStrategy determines the remediation strategy for an event. +func (r *RemediationController) determineStrategy(event *nvsentinelv1alpha1.HealthEvent) RemediationStrategy { + // Check for annotation override + if strategy, ok := event.Annotations["nvsentinel.nvidia.com/remediation-strategy"]; ok { + switch RemediationStrategy(strategy) { + case StrategyReboot, StrategyTerminate, StrategyGPUReset, StrategyManual, StrategyNone: + return RemediationStrategy(strategy) + } + } + + // Map recommended action to strategy + switch event.Spec.RecommendedAction { + case nvsentinelv1alpha1.ActionRestartVM, nvsentinelv1alpha1.ActionRestartBM: + return StrategyReboot + case nvsentinelv1alpha1.ActionReplaceVM: + return StrategyTerminate + case nvsentinelv1alpha1.ActionComponentReset: + return StrategyGPUReset + case nvsentinelv1alpha1.ActionNone: + return StrategyNone + case nvsentinelv1alpha1.ActionContactSupport, nvsentinelv1alpha1.ActionRunFieldDiag: + return StrategyManual + } + + // Use default strategy + if r.DefaultStrategy != "" { + return r.DefaultStrategy + } + + // Final fallback + return StrategyManual +} + +// executeReboot creates a privileged Job to reboot the node. +func (r *RemediationController) executeReboot(ctx context.Context, event *nvsentinelv1alpha1.HealthEvent, log klog.Logger) error { + nodeName := event.Spec.NodeName + + // Check if reboot job already exists + jobName := fmt.Sprintf("reboot-%s", event.Name) + var existingJob batchv1.Job + err := r.Get(ctx, types.NamespacedName{Name: jobName, Namespace: r.RebootJobNamespace}, &existingJob) + if err == nil { + // Job exists - check status + if existingJob.Status.Succeeded > 0 { + log.Info("Reboot job already succeeded", "job", jobName) + return nil + } + if existingJob.Status.Failed > 0 { + log.Info("Reboot job failed, will be retried", "job", jobName) + return fmt.Errorf("reboot job failed") + } + // Job still running - signal caller to requeue + log.Info("Reboot job still running", "job", jobName) + return errRebootInProgress + } + if !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to check existing reboot job: %w", err) + } + + // Determine image + image := r.RebootJobImage + if image == "" { + image = "busybox:latest" + } + + // Determine namespace + namespace := r.RebootJobNamespace + if namespace == "" { + namespace = "nvsentinel-system" + } + + // Calculate TTL + ttl := int32(r.RebootJobTTL.Seconds()) + if ttl == 0 { + ttl = 3600 // 1 hour default + } + + // Create reboot Job + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: jobName, + Namespace: namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "nvsentinel-reboot", + "app.kubernetes.io/component": "remediation", + "app.kubernetes.io/managed-by": "nvsentinel", + "nvsentinel.nvidia.com/event": event.Name, + "nvsentinel.nvidia.com/node": nodeName, + }, + Annotations: map[string]string{ + "nvsentinel.nvidia.com/health-event": event.Name, + }, + }, + Spec: batchv1.JobSpec{ + TTLSecondsAfterFinished: &ttl, + BackoffLimit: ptr.To[int32](3), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + NodeName: nodeName, + RestartPolicy: corev1.RestartPolicyNever, + HostPID: true, + Tolerations: []corev1.Toleration{ + { + Key: "node.kubernetes.io/unschedulable", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "nvidia.com/gpu-unhealthy", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "nvsentinel.nvidia.com/quarantine", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, + }, + Containers: []corev1.Container{ + { + Name: "reboot", + Image: image, + Command: []string{ + "/bin/sh", + "-c", + "echo 'Initiating reboot...' && nsenter -t 1 -m -u -i -n -- /sbin/reboot", + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + }, + }, + }, + }, + }, + }, + } + + if err := r.Create(ctx, job); err != nil { + return fmt.Errorf("failed to create reboot job: %w", err) + } + + log.Info("Created reboot job", "job", jobName, "namespace", namespace) + + // Record event + r.Recorder.Event(event, corev1.EventTypeNormal, "RebootJobCreated", + fmt.Sprintf("Created reboot job %s/%s", namespace, jobName)) + + return nil +} + +// executeTerminate deletes the Node object to trigger cloud provider replacement. +func (r *RemediationController) executeTerminate(ctx context.Context, event *nvsentinelv1alpha1.HealthEvent, log klog.Logger) error { + nodeName := event.Spec.NodeName + + // Get the node + var node corev1.Node + if err := r.Get(ctx, types.NamespacedName{Name: nodeName}, &node); err != nil { + if apierrors.IsNotFound(err) { + log.Info("Node already deleted/terminated", "node", nodeName) + return nil + } + return fmt.Errorf("failed to get node: %w", err) + } + + // Check if node has cloud provider info + if node.Spec.ProviderID == "" { + log.Info("Node has no provider ID, cannot terminate via cloud provider", "node", nodeName) + return fmt.Errorf("node %s has no provider ID - cannot use terminate strategy", nodeName) + } + + // Delete the node - cloud controller manager will handle instance termination + log.Info("Deleting node to trigger cloud provider termination", + "node", nodeName, + "providerID", node.Spec.ProviderID, + ) + + if err := r.Delete(ctx, &node); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("failed to delete node: %w", err) + } + + // Record event + r.Recorder.Event(event, corev1.EventTypeNormal, "NodeTerminated", + fmt.Sprintf("Deleted node %s to trigger cloud provider replacement", nodeName)) + + return nil +} + +// executeGPUReset initiates a GPU reset. +// This requires the NVML provider to be running on the node. +func (r *RemediationController) executeGPUReset(ctx context.Context, event *nvsentinelv1alpha1.HealthEvent, log klog.Logger) error { + nodeName := event.Spec.NodeName + + // Use a patch to set annotations, avoiding full object Update conflicts. + patch := client.MergeFrom(event.DeepCopy()) + + if event.Annotations == nil { + event.Annotations = make(map[string]string) + } + event.Annotations["nvsentinel.nvidia.com/gpu-reset-requested"] = "true" + event.Annotations["nvsentinel.nvidia.com/gpu-reset-requested-at"] = time.Now().UTC().Format(time.RFC3339) + + if err := r.Patch(ctx, event, patch); err != nil { + return fmt.Errorf("failed to patch event with gpu-reset annotation: %w", err) + } + + log.Info("GPU reset requested via annotation", "node", nodeName) + + // Record event + r.Recorder.Event(event, corev1.EventTypeNormal, "GPUResetRequested", + fmt.Sprintf("GPU reset requested on node %s", nodeName)) + + return nil +} + +// transitionToRemediated updates the HealthEvent status to Remediated phase. +func (r *RemediationController) transitionToRemediated(ctx context.Context, event *nvsentinelv1alpha1.HealthEvent, message string, log klog.Logger) error { + patch := client.MergeFrom(event.DeepCopy()) + now := metav1.Now() + + // Update status + event.Status.Phase = nvsentinelv1alpha1.PhaseRemediated + event.Status.LastRemediationTime = &now + event.Status.ObservedGeneration = event.Generation + + // Set condition + SetCondition(&event.Status.Conditions, nvsentinelv1alpha1.HealthEventCondition{ + Type: nvsentinelv1alpha1.ConditionRemediated, + Status: metav1.ConditionTrue, + LastTransitionTime: now, + Reason: "RemediationComplete", + Message: message, + ObservedGeneration: event.Generation, + }) + + if err := r.Status().Patch(ctx, event, patch); err != nil { + log.Error(err, "Failed to update HealthEvent status") + return err + } + + log.Info("HealthEvent transitioned to Remediated", + "healthEvent", event.Name, + "nodeName", event.Spec.NodeName, + "message", message, + ) + + // Record event + r.Recorder.Event(event, corev1.EventTypeNormal, "Remediated", message) + + return nil +} + +// SetupWithManager sets up the controller with the Manager. +// Note: We intentionally do NOT use phase-based predicates because status subresource +// updates may not trigger watch events reliably. Instead, we filter by phase in Reconcile. +func (r *RemediationController) SetupWithManager(mgr ctrl.Manager) error { + // Register metrics + registerRemediationMetrics() + + maxConcurrent := r.MaxConcurrentReconciles + if maxConcurrent == 0 { + maxConcurrent = 2 + } + + if r.DefaultStrategy == "" { + r.DefaultStrategy = StrategyManual + } + + return ctrl.NewControllerManagedBy(mgr). + For(&nvsentinelv1alpha1.HealthEvent{}). + WithOptions(controller.Options{ + MaxConcurrentReconciles: maxConcurrent, + }). + Named("remediation"). + Complete(r) +} diff --git a/pkg/controllers/healthevents/remediation_controller_test.go b/pkg/controllers/healthevents/remediation_controller_test.go new file mode 100644 index 000000000..84b7c752f --- /dev/null +++ b/pkg/controllers/healthevents/remediation_controller_test.go @@ -0,0 +1,536 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package healthevents + +import ( + "context" + "testing" + "time" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + nvsentinelv1alpha1 "github.com/nvidia/nvsentinel/api/nvsentinel/v1alpha1" +) + +func TestRemediationController_Reconcile(t *testing.T) { + scheme := runtime.NewScheme() + _ = nvsentinelv1alpha1.AddToScheme(scheme) + _ = corev1.AddToScheme(scheme) + _ = batchv1.AddToScheme(scheme) + + now := metav1.Now() + + tests := []struct { + name string + healthEvent *nvsentinelv1alpha1.HealthEvent + node *corev1.Node + existingJob *batchv1.Job + defaultStrategy RemediationStrategy + wantPhase nvsentinelv1alpha1.HealthEventPhase + wantSkip bool + wantRebootJob bool + wantNodeDeleted bool + wantAnnotation string + wantRequeue bool + }{ + { + name: "drained event with manual strategy should transition to remediated", + healthEvent: &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-remediation-1", + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "test", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "test-check", + IsFatal: true, + DetectedAt: now, + RecommendedAction: nvsentinelv1alpha1.ActionContactSupport, + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseDrained, + }, + }, + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-1", + }, + }, + defaultStrategy: StrategyManual, + wantPhase: nvsentinelv1alpha1.PhaseRemediated, + }, + { + name: "event not in drained phase should be skipped", + healthEvent: &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-remediation-2", + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "test", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "test-check", + IsFatal: true, + DetectedAt: now, + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseQuarantined, + }, + }, + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-1", + }, + }, + defaultStrategy: StrategyManual, + wantPhase: nvsentinelv1alpha1.PhaseQuarantined, + wantSkip: true, + }, + { + name: "drained event with annotation override should use specified strategy", + healthEvent: &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-remediation-3", + Annotations: map[string]string{ + "nvsentinel.nvidia.com/remediation-strategy": "none", + }, + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "test", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "test-check", + IsFatal: true, + DetectedAt: now, + RecommendedAction: nvsentinelv1alpha1.ActionRestartVM, // Would normally be reboot + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseDrained, + }, + }, + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-1", + }, + }, + defaultStrategy: StrategyReboot, + wantPhase: nvsentinelv1alpha1.PhaseRemediated, + }, + { + name: "drained event with GPU reset action should request reset", + healthEvent: &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-remediation-4", + Annotations: map[string]string{}, + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "test", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "test-check", + IsFatal: false, + DetectedAt: now, + RecommendedAction: nvsentinelv1alpha1.ActionComponentReset, + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseDrained, + }, + }, + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-1", + }, + }, + defaultStrategy: StrategyManual, + wantPhase: nvsentinelv1alpha1.PhaseRemediated, + wantAnnotation: "nvsentinel.nvidia.com/gpu-reset-requested", + }, + { + name: "drained event with reboot action should create reboot job", + healthEvent: &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-remediation-5", + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "test", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "test-check", + IsFatal: true, + DetectedAt: now, + RecommendedAction: nvsentinelv1alpha1.ActionRestartVM, + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseDrained, + }, + }, + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-1", + }, + }, + defaultStrategy: StrategyManual, + wantPhase: nvsentinelv1alpha1.PhaseRemediated, + wantRebootJob: true, + }, + { + name: "drained event with terminate action and providerID should delete node", + healthEvent: &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-remediation-6", + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "test", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "test-check", + IsFatal: true, + DetectedAt: now, + RecommendedAction: nvsentinelv1alpha1.ActionReplaceVM, + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseDrained, + }, + }, + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-1", + }, + Spec: corev1.NodeSpec{ + ProviderID: "aws:///us-west-2a/i-1234567890abcdef0", + }, + }, + defaultStrategy: StrategyManual, + wantPhase: nvsentinelv1alpha1.PhaseRemediated, + wantNodeDeleted: true, + }, + { + name: "drained event with running reboot job should requeue", + healthEvent: &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-reboot-running", + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + NodeName: "node-1", + Source: "test", + ComponentClass: "GPU", + CheckName: "xid-check", + IsFatal: true, + DetectedAt: metav1.Now(), + RecommendedAction: nvsentinelv1alpha1.ActionRestartBM, + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseDrained, + }, + }, + existingJob: &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "reboot-test-reboot-running", + Namespace: "nvsentinel-system", + }, + Status: batchv1.JobStatus{ + Active: 1, + }, + }, + wantPhase: nvsentinelv1alpha1.PhaseDrained, // Should NOT transition + wantRequeue: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + objects := []runtime.Object{tt.healthEvent} + if tt.node != nil { + objects = append(objects, tt.node) + } + if tt.existingJob != nil { + objects = append(objects, tt.existingJob) + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(objects...). + WithStatusSubresource(&nvsentinelv1alpha1.HealthEvent{}). + Build() + + recorder := record.NewFakeRecorder(10) + + r := &RemediationController{ + Client: fakeClient, + Scheme: scheme, + Recorder: recorder, + DefaultStrategy: tt.defaultStrategy, + RebootJobNamespace: "nvsentinel-system", + RebootJobImage: "busybox:latest", + RebootJobTTL: time.Hour, + } + + ctx := context.Background() + req := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: tt.healthEvent.Name, + }, + } + + result, err := r.Reconcile(ctx, req) + if err != nil { + t.Errorf("Reconcile() error = %v", err) + return + } + + // Verify requeue behavior + if tt.wantRequeue { + if result.RequeueAfter == 0 && !result.Requeue { + t.Errorf("Expected requeue, but got result = %+v", result) + } + } + + // Verify phase + var event nvsentinelv1alpha1.HealthEvent + if err := fakeClient.Get(ctx, req.NamespacedName, &event); err != nil { + t.Fatalf("Failed to get HealthEvent: %v", err) + } + + if event.Status.Phase != tt.wantPhase { + t.Errorf("Phase = %v, want %v", event.Status.Phase, tt.wantPhase) + } + + // Verify annotation if expected + if tt.wantAnnotation != "" { + if _, ok := event.Annotations[tt.wantAnnotation]; !ok { + t.Errorf("Expected annotation %s not found", tt.wantAnnotation) + } + } + + // Verify reboot job if expected + if tt.wantRebootJob { + var job batchv1.Job + jobName := "reboot-" + tt.healthEvent.Name + err := fakeClient.Get(ctx, types.NamespacedName{ + Name: jobName, + Namespace: "nvsentinel-system", + }, &job) + if err != nil { + t.Errorf("Expected reboot job %s not found: %v", jobName, err) + } + } + + // Verify node deletion if expected + if tt.wantNodeDeleted && tt.node != nil { + var node corev1.Node + err := fakeClient.Get(ctx, types.NamespacedName{Name: tt.node.Name}, &node) + if err == nil { + t.Error("Expected node to be deleted, but it still exists") + } + } + }) + } +} + +func TestRemediationController_DetermineStrategy(t *testing.T) { + tests := []struct { + name string + event *nvsentinelv1alpha1.HealthEvent + defaultStrategy RemediationStrategy + wantStrategy RemediationStrategy + }{ + { + name: "annotation override takes precedence", + event: &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "nvsentinel.nvidia.com/remediation-strategy": "reboot", + }, + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + RecommendedAction: nvsentinelv1alpha1.ActionNone, + }, + }, + defaultStrategy: StrategyManual, + wantStrategy: StrategyReboot, + }, + { + name: "RestartVM maps to reboot", + event: &nvsentinelv1alpha1.HealthEvent{ + Spec: nvsentinelv1alpha1.HealthEventSpec{ + RecommendedAction: nvsentinelv1alpha1.ActionRestartVM, + }, + }, + defaultStrategy: StrategyManual, + wantStrategy: StrategyReboot, + }, + { + name: "RestartBM maps to reboot", + event: &nvsentinelv1alpha1.HealthEvent{ + Spec: nvsentinelv1alpha1.HealthEventSpec{ + RecommendedAction: nvsentinelv1alpha1.ActionRestartBM, + }, + }, + defaultStrategy: StrategyManual, + wantStrategy: StrategyReboot, + }, + { + name: "ReplaceVM maps to terminate", + event: &nvsentinelv1alpha1.HealthEvent{ + Spec: nvsentinelv1alpha1.HealthEventSpec{ + RecommendedAction: nvsentinelv1alpha1.ActionReplaceVM, + }, + }, + defaultStrategy: StrategyManual, + wantStrategy: StrategyTerminate, + }, + { + name: "ComponentReset maps to gpu-reset", + event: &nvsentinelv1alpha1.HealthEvent{ + Spec: nvsentinelv1alpha1.HealthEventSpec{ + RecommendedAction: nvsentinelv1alpha1.ActionComponentReset, + }, + }, + defaultStrategy: StrategyManual, + wantStrategy: StrategyGPUReset, + }, + { + name: "ActionNone maps to none", + event: &nvsentinelv1alpha1.HealthEvent{ + Spec: nvsentinelv1alpha1.HealthEventSpec{ + RecommendedAction: nvsentinelv1alpha1.ActionNone, + }, + }, + defaultStrategy: StrategyManual, + wantStrategy: StrategyNone, + }, + { + name: "ContactSupport maps to manual", + event: &nvsentinelv1alpha1.HealthEvent{ + Spec: nvsentinelv1alpha1.HealthEventSpec{ + RecommendedAction: nvsentinelv1alpha1.ActionContactSupport, + }, + }, + defaultStrategy: StrategyReboot, + wantStrategy: StrategyManual, + }, + { + name: "Unknown action uses default", + event: &nvsentinelv1alpha1.HealthEvent{ + Spec: nvsentinelv1alpha1.HealthEventSpec{ + RecommendedAction: nvsentinelv1alpha1.ActionUnknown, + }, + }, + defaultStrategy: StrategyReboot, + wantStrategy: StrategyReboot, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &RemediationController{ + DefaultStrategy: tt.defaultStrategy, + } + + got := r.determineStrategy(tt.event) + if got != tt.wantStrategy { + t.Errorf("determineStrategy() = %v, want %v", got, tt.wantStrategy) + } + }) + } +} + +func TestRemediationController_TerminateWithoutProviderID(t *testing.T) { + scheme := runtime.NewScheme() + _ = nvsentinelv1alpha1.AddToScheme(scheme) + _ = corev1.AddToScheme(scheme) + _ = batchv1.AddToScheme(scheme) + + now := metav1.Now() + + // Event that would trigger terminate, but node has no providerID + healthEvent := &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-no-provider", + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "test", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "test-check", + IsFatal: true, + DetectedAt: now, + RecommendedAction: nvsentinelv1alpha1.ActionReplaceVM, + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseDrained, + }, + } + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-1", + }, + // No ProviderID set + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(healthEvent, node). + WithStatusSubresource(&nvsentinelv1alpha1.HealthEvent{}). + Build() + + recorder := record.NewFakeRecorder(10) + + r := &RemediationController{ + Client: fakeClient, + Scheme: scheme, + Recorder: recorder, + DefaultStrategy: StrategyManual, + } + + ctx := context.Background() + req := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: healthEvent.Name, + }, + } + + // Should not error (requeues for retry) + _, err := r.Reconcile(ctx, req) + if err != nil { + t.Errorf("Reconcile() unexpected error = %v", err) + } + + // Event should NOT be in Remediated phase (remediation failed) + var event nvsentinelv1alpha1.HealthEvent + if err := fakeClient.Get(ctx, req.NamespacedName, &event); err != nil { + t.Fatalf("Failed to get HealthEvent: %v", err) + } + + // Phase should still be Drained (remediation failed) + if event.Status.Phase != nvsentinelv1alpha1.PhaseDrained { + t.Errorf("Phase = %v, want Drained (remediation should fail)", event.Status.Phase) + } + + // Should have a failed condition + cond := GetCondition(event.Status.Conditions, nvsentinelv1alpha1.ConditionRemediated) + if cond == nil || cond.Status != metav1.ConditionFalse { + t.Error("Expected Remediated condition to be False") + } +} diff --git a/pkg/controllers/healthevents/ttl_controller.go b/pkg/controllers/healthevents/ttl_controller.go new file mode 100644 index 000000000..f3aadd4d8 --- /dev/null +++ b/pkg/controllers/healthevents/ttl_controller.go @@ -0,0 +1,360 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package healthevents + +import ( + "context" + "strconv" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + + nvsentinelv1alpha1 "github.com/nvidia/nvsentinel/api/nvsentinel/v1alpha1" +) + +// TTLController reconciles HealthEvent resources to delete them after TTL expiry. +// It watches for events in Resolved or Cancelled phase and deletes them after +// the configured TTL period has elapsed. +type TTLController struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder + + // PolicyConfigMapName is the name of the ConfigMap containing TTL policy. + PolicyConfigMapName string + + // PolicyConfigMapNamespace is the namespace of the policy ConfigMap. + PolicyConfigMapNamespace string + + // DefaultTTL is the default TTL if ConfigMap is not found. + DefaultTTL time.Duration + + // MaxConcurrentReconciles is the maximum number of concurrent Reconciles. + MaxConcurrentReconciles int + + // BatchSize is the maximum number of events to delete per cycle. + BatchSize int +} + +// TTLPolicy holds the parsed TTL configuration. +type TTLPolicy struct { + TTLAfterResolved time.Duration + RetentionMode string // "standard" or "compliance" + ComplianceRetention time.Duration + BatchSize int + CleanupInterval time.Duration + MaxEventsPerNode int + MaxTotalEvents int +} + +// DefaultTTLPolicy returns the default TTL policy. +func DefaultTTLPolicy() TTLPolicy { + return TTLPolicy{ + TTLAfterResolved: 168 * time.Hour, // 7 days + RetentionMode: "standard", + ComplianceRetention: 2160 * time.Hour, // 90 days + BatchSize: 100, + CleanupInterval: 5 * time.Minute, + MaxEventsPerNode: 0, // unlimited + MaxTotalEvents: 0, // unlimited + } +} + +// +kubebuilder:rbac:groups=nvsentinel.nvidia.com,resources=healthevents,verbs=get;list;watch;delete +// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch + +// Reconcile handles HealthEvent TTL cleanup. +// It checks if the event has exceeded its TTL and deletes it if so. +func (r *TTLController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := klog.FromContext(ctx).WithValues("healthevent", req.Name) + log.V(2).Info("Reconciling HealthEvent for TTL") + + // Fetch the HealthEvent + var healthEvent nvsentinelv1alpha1.HealthEvent + if err := r.Get(ctx, req.NamespacedName, &healthEvent); err != nil { + if apierrors.IsNotFound(err) { + // Already deleted + return ctrl.Result{}, nil + } + log.Error(err, "Failed to get HealthEvent") + return ctrl.Result{}, err + } + + // Only process resolved or cancelled events + if healthEvent.Status.Phase != nvsentinelv1alpha1.PhaseResolved && + healthEvent.Status.Phase != nvsentinelv1alpha1.PhaseCancelled { + log.V(2).Info("Skipping event not in terminal phase", "phase", healthEvent.Status.Phase) + return ctrl.Result{}, nil + } + + // Get TTL policy + policy := r.getPolicy(ctx) + + // Calculate effective TTL + ttl := policy.TTLAfterResolved + if policy.RetentionMode == "compliance" { + ttl = policy.ComplianceRetention + } + + // TTL of 0 means disabled + if ttl == 0 { + log.V(2).Info("TTL cleanup disabled") + return ctrl.Result{}, nil + } + + // Check if event has expired + resolvedAt := healthEvent.Status.ResolvedAt + if resolvedAt == nil { + // Use creation time if resolvedAt not set + resolvedAt = &healthEvent.CreationTimestamp + } + + age := time.Since(resolvedAt.Time) + if age < ttl { + // Not yet expired, requeue for later + requeueAfter := ttl - age + time.Minute // Add buffer + log.V(2).Info("Event not yet expired, requeueing", + "age", age.Round(time.Second), + "ttl", ttl, + "requeueAfter", requeueAfter.Round(time.Second), + ) + return ctrl.Result{RequeueAfter: requeueAfter}, nil + } + + // Delete the event + log.Info("Deleting expired HealthEvent", + "age", age.Round(time.Second), + "ttl", ttl, + "nodeName", healthEvent.Spec.NodeName, + ) + + if err := r.Delete(ctx, &healthEvent); err != nil { + if apierrors.IsNotFound(err) { + // Already deleted + return ctrl.Result{}, nil + } + log.Error(err, "Failed to delete HealthEvent") + return ctrl.Result{}, err + } + + // Record metrics + ttlDeletionsTotal.WithLabelValues(healthEvent.Spec.NodeName, string(healthEvent.Status.Phase)).Inc() + + log.Info("Deleted expired HealthEvent", "name", healthEvent.Name) + return ctrl.Result{}, nil +} + +// getPolicy reads the TTL policy from ConfigMap or returns defaults. +func (r *TTLController) getPolicy(ctx context.Context) TTLPolicy { + log := klog.FromContext(ctx) + policy := DefaultTTLPolicy() + + // Override with controller's DefaultTTL if set + if r.DefaultTTL != 0 { + policy.TTLAfterResolved = r.DefaultTTL + } + + if r.PolicyConfigMapName == "" || r.PolicyConfigMapNamespace == "" { + return policy + } + + var cm corev1.ConfigMap + err := r.Get(ctx, types.NamespacedName{ + Name: r.PolicyConfigMapName, + Namespace: r.PolicyConfigMapNamespace, + }, &cm) + if err != nil { + if !apierrors.IsNotFound(err) { + log.Error(err, "Failed to get policy ConfigMap, using defaults") + } + return policy + } + + // Parse TTL after resolved + if val, ok := cm.Data["ttlAfterResolved"]; ok { + if d, err := time.ParseDuration(val); err == nil { + policy.TTLAfterResolved = d + } else { + log.V(1).Info("Invalid ttlAfterResolved, using default", "value", val) + } + } + + // Parse retention mode + if val, ok := cm.Data["retentionMode"]; ok { + if val == "standard" || val == "compliance" { + policy.RetentionMode = val + } + } + + // Parse compliance retention + if val, ok := cm.Data["complianceRetention"]; ok { + if d, err := time.ParseDuration(val); err == nil { + policy.ComplianceRetention = d + } + } + + // Parse batch size + if val, ok := cm.Data["batchSize"]; ok { + if n, err := strconv.Atoi(val); err == nil && n > 0 { + policy.BatchSize = n + } + } + + // Parse cleanup interval + if val, ok := cm.Data["cleanupInterval"]; ok { + if d, err := time.ParseDuration(val); err == nil { + policy.CleanupInterval = d + } + } + + // Parse max events per node + if val, ok := cm.Data["maxEventsPerNode"]; ok { + if n, err := strconv.Atoi(val); err == nil && n >= 0 { + policy.MaxEventsPerNode = n + } + } + + // Parse max total events + if val, ok := cm.Data["maxTotalEvents"]; ok { + if n, err := strconv.Atoi(val); err == nil && n >= 0 { + policy.MaxTotalEvents = n + } + } + + return policy +} + +// SetupWithManager sets up the controller with the Manager. +// Note: We intentionally do NOT use phase-based predicates because status subresource +// updates may not trigger watch events reliably. Instead, we filter by phase in Reconcile. +func (r *TTLController) SetupWithManager(mgr ctrl.Manager) error { + // Register metrics + registerTTLMetrics() + + maxConcurrent := r.MaxConcurrentReconciles + if maxConcurrent == 0 { + maxConcurrent = 1 // TTL cleanup is not latency-sensitive + } + + if r.DefaultTTL == 0 { + r.DefaultTTL = 168 * time.Hour // 7 days + } + + return ctrl.NewControllerManagedBy(mgr). + For(&nvsentinelv1alpha1.HealthEvent{}). + WithOptions(controller.Options{ + MaxConcurrentReconciles: maxConcurrent, + }). + Named("ttl"). + Complete(r) +} + +// RunPeriodicCleanup runs periodic cleanup of old events. +// This is useful for catching events that might have been missed. +func (r *TTLController) RunPeriodicCleanup(ctx context.Context) { + log := klog.FromContext(ctx).WithName("ttl-cleanup") + + // Get initial policy for interval + policy := r.getPolicy(ctx) + ticker := time.NewTicker(policy.CleanupInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + r.cleanupExpiredEvents(ctx, log) + + // Update ticker if policy changed + newPolicy := r.getPolicy(ctx) + if newPolicy.CleanupInterval != policy.CleanupInterval { + ticker.Reset(newPolicy.CleanupInterval) + policy = newPolicy + } + } + } +} + +// cleanupExpiredEvents lists and deletes expired events in batches. +func (r *TTLController) cleanupExpiredEvents(ctx context.Context, log klog.Logger) { + policy := r.getPolicy(ctx) + + // TTL of 0 means disabled + ttl := policy.TTLAfterResolved + if policy.RetentionMode == "compliance" { + ttl = policy.ComplianceRetention + } + if ttl == 0 { + return + } + + // List all resolved/cancelled events + var events nvsentinelv1alpha1.HealthEventList + if err := r.List(ctx, &events); err != nil { + log.Error(err, "Failed to list HealthEvents for cleanup") + return + } + + deleted := 0 + for i := range events.Items { + if deleted >= policy.BatchSize { + log.V(1).Info("Reached batch size limit", "deleted", deleted) + break + } + + event := &events.Items[i] + + // Skip non-terminal events + if event.Status.Phase != nvsentinelv1alpha1.PhaseResolved && + event.Status.Phase != nvsentinelv1alpha1.PhaseCancelled { + continue + } + + // Check expiry + resolvedAt := event.Status.ResolvedAt + if resolvedAt == nil { + resolvedAt = &event.CreationTimestamp + } + + if time.Since(resolvedAt.Time) < ttl { + continue + } + + // Delete + if err := r.Delete(ctx, event); err != nil { + if !apierrors.IsNotFound(err) { + log.Error(err, "Failed to delete expired event", "name", event.Name) + } + continue + } + + deleted++ + ttlDeletionsTotal.WithLabelValues(event.Spec.NodeName, string(event.Status.Phase)).Inc() + } + + if deleted > 0 { + log.Info("Periodic cleanup completed", "deleted", deleted) + } +} diff --git a/pkg/controllers/healthevents/ttl_controller_test.go b/pkg/controllers/healthevents/ttl_controller_test.go new file mode 100644 index 000000000..3fef3ebc2 --- /dev/null +++ b/pkg/controllers/healthevents/ttl_controller_test.go @@ -0,0 +1,439 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package healthevents + +import ( + "context" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + nvsentinelv1alpha1 "github.com/nvidia/nvsentinel/api/nvsentinel/v1alpha1" +) + +func TestTTLController_Reconcile(t *testing.T) { + scheme := runtime.NewScheme() + _ = nvsentinelv1alpha1.AddToScheme(scheme) + _ = corev1.AddToScheme(scheme) + + now := metav1.Now() + oldTime := metav1.NewTime(now.Add(-200 * time.Hour)) // 200 hours ago (past 168h default TTL) + recentTime := metav1.NewTime(now.Add(-1 * time.Hour)) // 1 hour ago + + tests := []struct { + name string + healthEvent *nvsentinelv1alpha1.HealthEvent + ttl time.Duration + wantDeleted bool + wantRequeue bool + }{ + { + name: "expired resolved event should be deleted", + healthEvent: &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ttl-1", + CreationTimestamp: oldTime, + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "test", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "test-check", + DetectedAt: oldTime, + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseResolved, + ResolvedAt: &oldTime, + }, + }, + ttl: 168 * time.Hour, + wantDeleted: true, + wantRequeue: false, + }, + { + name: "recent resolved event should be requeued", + healthEvent: &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ttl-2", + CreationTimestamp: recentTime, + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "test", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "test-check", + DetectedAt: recentTime, + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseResolved, + ResolvedAt: &recentTime, + }, + }, + ttl: 168 * time.Hour, + wantDeleted: false, + wantRequeue: true, + }, + { + name: "non-terminal event should be skipped", + healthEvent: &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ttl-3", + CreationTimestamp: oldTime, + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "test", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "test-check", + DetectedAt: oldTime, + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseQuarantined, + }, + }, + ttl: 168 * time.Hour, + wantDeleted: false, + wantRequeue: false, + }, + { + name: "expired cancelled event should be deleted", + healthEvent: &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ttl-4", + CreationTimestamp: oldTime, + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "test", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "test-check", + DetectedAt: oldTime, + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseCancelled, + ResolvedAt: &oldTime, + }, + }, + ttl: 168 * time.Hour, + wantDeleted: true, + wantRequeue: false, + }, + { + name: "very long TTL should not delete recent event", + healthEvent: &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ttl-5", + CreationTimestamp: recentTime, + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "test", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "test-check", + DetectedAt: recentTime, + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseResolved, + ResolvedAt: &recentTime, + }, + }, + ttl: 8760 * time.Hour, // 1 year - won't delete + wantDeleted: false, + wantRequeue: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(tt.healthEvent). + Build() + + recorder := record.NewFakeRecorder(10) + + r := &TTLController{ + Client: fakeClient, + Scheme: scheme, + Recorder: recorder, + DefaultTTL: tt.ttl, + } + + ctx := context.Background() + req := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: tt.healthEvent.Name, + }, + } + + result, err := r.Reconcile(ctx, req) + if err != nil { + t.Errorf("Reconcile() error = %v", err) + return + } + + // Check if event was deleted + var event nvsentinelv1alpha1.HealthEvent + err = fakeClient.Get(ctx, req.NamespacedName, &event) + + if tt.wantDeleted { + if !apierrors.IsNotFound(err) { + t.Error("Expected event to be deleted, but it still exists") + } + } else { + if apierrors.IsNotFound(err) { + t.Error("Expected event to exist, but it was deleted") + } + } + + // Check requeue + if tt.wantRequeue && result.RequeueAfter == 0 { + t.Error("Expected requeue, but RequeueAfter is 0") + } + if !tt.wantRequeue && result.RequeueAfter > 0 { + t.Errorf("Did not expect requeue, but RequeueAfter is %v", result.RequeueAfter) + } + }) + } +} + +func TestTTLController_GetPolicy(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + + tests := []struct { + name string + configMap *corev1.ConfigMap + wantTTL time.Duration + wantMode string + }{ + { + name: "no ConfigMap returns defaults", + configMap: nil, + wantTTL: 168 * time.Hour, + wantMode: "standard", + }, + { + name: "ConfigMap with custom TTL", + configMap: &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-policy", + Namespace: "test-ns", + }, + Data: map[string]string{ + "ttlAfterResolved": "720h", + "retentionMode": "standard", + }, + }, + wantTTL: 720 * time.Hour, + wantMode: "standard", + }, + { + name: "ConfigMap with compliance mode", + configMap: &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-policy", + Namespace: "test-ns", + }, + Data: map[string]string{ + "ttlAfterResolved": "168h", + "retentionMode": "compliance", + "complianceRetention": "2160h", + }, + }, + wantTTL: 168 * time.Hour, + wantMode: "compliance", + }, + { + name: "ConfigMap with zero TTL disables cleanup", + configMap: &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-policy", + Namespace: "test-ns", + }, + Data: map[string]string{ + "ttlAfterResolved": "0", + "retentionMode": "standard", + }, + }, + wantTTL: 0, + wantMode: "standard", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + builder := fake.NewClientBuilder().WithScheme(scheme) + if tt.configMap != nil { + builder = builder.WithObjects(tt.configMap) + } + fakeClient := builder.Build() + + r := &TTLController{ + Client: fakeClient, + PolicyConfigMapName: "test-policy", + PolicyConfigMapNamespace: "test-ns", + } + + ctx := context.Background() + policy := r.getPolicy(ctx) + + if policy.TTLAfterResolved != tt.wantTTL { + t.Errorf("getPolicy() TTL = %v, want %v", policy.TTLAfterResolved, tt.wantTTL) + } + if policy.RetentionMode != tt.wantMode { + t.Errorf("getPolicy() Mode = %v, want %v", policy.RetentionMode, tt.wantMode) + } + }) + } +} + +func TestDefaultTTLPolicy(t *testing.T) { + policy := DefaultTTLPolicy() + + if policy.TTLAfterResolved != 168*time.Hour { + t.Errorf("DefaultTTLPolicy() TTLAfterResolved = %v, want 168h", policy.TTLAfterResolved) + } + if policy.RetentionMode != "standard" { + t.Errorf("DefaultTTLPolicy() RetentionMode = %v, want standard", policy.RetentionMode) + } + if policy.ComplianceRetention != 2160*time.Hour { + t.Errorf("DefaultTTLPolicy() ComplianceRetention = %v, want 2160h", policy.ComplianceRetention) + } + if policy.BatchSize != 100 { + t.Errorf("DefaultTTLPolicy() BatchSize = %v, want 100", policy.BatchSize) + } + if policy.CleanupInterval != 5*time.Minute { + t.Errorf("DefaultTTLPolicy() CleanupInterval = %v, want 5m", policy.CleanupInterval) + } +} + +func TestTTLController_CleanupExpiredEvents(t *testing.T) { + scheme := runtime.NewScheme() + _ = nvsentinelv1alpha1.AddToScheme(scheme) + _ = corev1.AddToScheme(scheme) + + oldTime := metav1.NewTime(time.Now().Add(-200 * time.Hour)) + recentTime := metav1.NewTime(time.Now().Add(-1 * time.Hour)) + + // Create mix of events + events := []client.Object{ + &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "expired-1", + CreationTimestamp: oldTime, + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "test", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "test", + DetectedAt: oldTime, + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseResolved, + ResolvedAt: &oldTime, + }, + }, + &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "expired-2", + CreationTimestamp: oldTime, + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "test", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "test", + DetectedAt: oldTime, + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseCancelled, + ResolvedAt: &oldTime, + }, + }, + &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "recent-1", + CreationTimestamp: recentTime, + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "test", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "test", + DetectedAt: recentTime, + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseResolved, + ResolvedAt: &recentTime, + }, + }, + &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "active-1", + CreationTimestamp: oldTime, + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "test", + NodeName: "worker-1", + ComponentClass: "GPU", + CheckName: "test", + DetectedAt: oldTime, + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseQuarantined, + }, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(events...). + Build() + + r := &TTLController{ + Client: fakeClient, + DefaultTTL: 168 * time.Hour, + } + + ctx := context.Background() + log := ctrl.Log.WithName("test") + + r.cleanupExpiredEvents(ctx, log) + + // Verify expired events were deleted + var remaining nvsentinelv1alpha1.HealthEventList + if err := fakeClient.List(ctx, &remaining); err != nil { + t.Fatalf("Failed to list events: %v", err) + } + + // Should have 2 remaining: recent-1 (not expired) and active-1 (not terminal) + if len(remaining.Items) != 2 { + t.Errorf("Expected 2 remaining events, got %d", len(remaining.Items)) + for _, e := range remaining.Items { + t.Logf("Remaining: %s (phase=%s)", e.Name, e.Status.Phase) + } + } +} diff --git a/pkg/deviceapiserver/cache/broadcaster.go b/pkg/deviceapiserver/cache/broadcaster.go new file mode 100644 index 000000000..29f3bd2f8 --- /dev/null +++ b/pkg/deviceapiserver/cache/broadcaster.go @@ -0,0 +1,168 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cache + +import ( + "sync" + + "k8s.io/klog/v2" + + v1alpha1 "github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1" +) + +// Event types for watch notifications. +const ( + EventTypeAdded = "ADDED" + EventTypeModified = "MODIFIED" + EventTypeDeleted = "DELETED" + EventTypeError = "ERROR" +) + +// WatchEvent represents a change event for a GPU resource. +type WatchEvent struct { + // Type indicates the nature of the change. + Type string + + // Object is the GPU resource. + Object *v1alpha1.Gpu +} + +// subscriber represents a watch subscriber. +type subscriber struct { + id string + channel chan WatchEvent +} + +// Broadcaster manages watch event broadcasting to multiple subscribers. +// +// Events are sent to all active subscribers. If a subscriber's buffer is full, +// the event is dropped for that subscriber (non-blocking send). +type Broadcaster struct { + mu sync.RWMutex + subscribers map[string]*subscriber + bufferSize int + logger klog.Logger + onEventDrop func() // Optional callback when events are dropped +} + +// NewBroadcaster creates a new Broadcaster instance. +func NewBroadcaster(logger klog.Logger, bufferSize int) *Broadcaster { + if bufferSize <= 0 { + bufferSize = 100 + } + + return &Broadcaster{ + subscribers: make(map[string]*subscriber), + bufferSize: bufferSize, + logger: logger, + } +} + +// SetOnEventDrop sets a callback function that will be called when an event +// is dropped due to a full subscriber buffer. This is useful for metrics. +func (b *Broadcaster) SetOnEventDrop(fn func()) { + b.mu.Lock() + defer b.mu.Unlock() + b.onEventDrop = fn +} + +// Subscribe creates a new subscription and returns a channel to receive events. +// +// The returned channel has a buffer to prevent blocking the broadcaster. +// If the buffer fills up, events will be dropped for this subscriber. +// +// The caller must call Unsubscribe when done to clean up resources. +func (b *Broadcaster) Subscribe(id string) <-chan WatchEvent { + b.mu.Lock() + defer b.mu.Unlock() + + // Close existing subscription if any + if existing, ok := b.subscribers[id]; ok { + close(existing.channel) + delete(b.subscribers, id) + } + + ch := make(chan WatchEvent, b.bufferSize) + b.subscribers[id] = &subscriber{ + id: id, + channel: ch, + } + + b.logger.V(2).Info("Subscriber added", "id", id, "totalSubscribers", len(b.subscribers)) + + return ch +} + +// Unsubscribe removes a subscription and closes its channel. +// +// After calling Unsubscribe, the channel returned by Subscribe will be closed. +func (b *Broadcaster) Unsubscribe(id string) { + b.mu.Lock() + defer b.mu.Unlock() + + if sub, ok := b.subscribers[id]; ok { + close(sub.channel) + delete(b.subscribers, id) + b.logger.V(2).Info("Subscriber removed", "id", id, "totalSubscribers", len(b.subscribers)) + } +} + +// Notify sends an event to all subscribers. +// +// This is a non-blocking operation. If a subscriber's buffer is full, +// the event is dropped for that subscriber and the onEventDrop callback +// is invoked (if set). +func (b *Broadcaster) Notify(event WatchEvent) { + b.mu.RLock() + defer b.mu.RUnlock() + + for _, sub := range b.subscribers { + select { + case sub.channel <- event: + // Event sent successfully + default: + // Buffer full, drop event for this subscriber + b.logger.V(1).Info("Event dropped due to full buffer", + "subscriberID", sub.id, + "eventType", event.Type, + "gpuName", event.Object.GetMetadata().GetName(), + ) + if b.onEventDrop != nil { + b.onEventDrop() + } + } + } +} + +// SubscriberCount returns the number of active subscribers. +func (b *Broadcaster) SubscriberCount() int { + b.mu.RLock() + defer b.mu.RUnlock() + + return len(b.subscribers) +} + +// Close closes all subscriber channels and removes all subscriptions. +func (b *Broadcaster) Close() { + b.mu.Lock() + defer b.mu.Unlock() + + for id, sub := range b.subscribers { + close(sub.channel) + delete(b.subscribers, id) + } + + b.logger.V(1).Info("Broadcaster closed") +} diff --git a/pkg/deviceapiserver/cache/broadcaster_test.go b/pkg/deviceapiserver/cache/broadcaster_test.go new file mode 100644 index 000000000..c26ee13d4 --- /dev/null +++ b/pkg/deviceapiserver/cache/broadcaster_test.go @@ -0,0 +1,231 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cache + +import ( + "sync" + "testing" + "time" + + "k8s.io/klog/v2" + + v1alpha1 "github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1" +) + +func TestBroadcaster_SubscribeUnsubscribe(t *testing.T) { + logger := klog.Background() + b := NewBroadcaster(logger, 10) + + // Subscribe + ch := b.Subscribe("sub-1") + if ch == nil { + t.Fatal("Subscribe returned nil channel") + } + + if b.SubscriberCount() != 1 { + t.Errorf("Expected 1 subscriber, got %d", b.SubscriberCount()) + } + + // Unsubscribe + b.Unsubscribe("sub-1") + + if b.SubscriberCount() != 0 { + t.Errorf("Expected 0 subscribers, got %d", b.SubscriberCount()) + } + + // Channel should be closed + select { + case _, ok := <-ch: + if ok { + t.Error("Channel should be closed after unsubscribe") + } + default: + t.Error("Channel should be closed and readable") + } +} + +func TestBroadcaster_Notify(t *testing.T) { + logger := klog.Background() + b := NewBroadcaster(logger, 10) + + // Subscribe two subscribers + ch1 := b.Subscribe("sub-1") + ch2 := b.Subscribe("sub-2") + + // Send event + gpu := &v1alpha1.Gpu{Metadata: &v1alpha1.ObjectMeta{Name: "gpu-0"}} + event := WatchEvent{Type: EventTypeAdded, Object: gpu} + b.Notify(event) + + // Both should receive + select { + case e := <-ch1: + if e.Type != EventTypeAdded || e.Object.GetMetadata().GetName() != "gpu-0" { + t.Errorf("Unexpected event: %+v", e) + } + case <-time.After(time.Second): + t.Error("Subscriber 1 did not receive event") + } + + select { + case e := <-ch2: + if e.Type != EventTypeAdded || e.Object.GetMetadata().GetName() != "gpu-0" { + t.Errorf("Unexpected event: %+v", e) + } + case <-time.After(time.Second): + t.Error("Subscriber 2 did not receive event") + } +} + +func TestBroadcaster_SlowSubscriberDoesNotBlock(t *testing.T) { + logger := klog.Background() + b := NewBroadcaster(logger, 2) // Small buffer + + // Subscribe - don't read from channel + _ = b.Subscribe("slow-sub") + fastCh := b.Subscribe("fast-sub") + + // Send more events than buffer size + for i := 0; i < 10; i++ { + gpu := &v1alpha1.Gpu{Metadata: &v1alpha1.ObjectMeta{Name: "gpu-0"}} + b.Notify(WatchEvent{Type: EventTypeModified, Object: gpu}) + } + + // Fast subscriber should receive events (up to buffer) + received := 0 + for { + select { + case <-fastCh: + received++ + default: + goto done + } + } +done: + + if received == 0 { + t.Error("Fast subscriber should have received some events") + } + if received > 2 { + t.Errorf("Fast subscriber received more than buffer size: %d", received) + } +} + +func TestBroadcaster_ConcurrentNotify(t *testing.T) { + logger := klog.Background() + b := NewBroadcaster(logger, 1000) + + // Subscribe + ch := b.Subscribe("sub-1") + + // Concurrent writers + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func(id int) { + defer wg.Done() + for j := 0; j < 100; j++ { + gpu := &v1alpha1.Gpu{Metadata: &v1alpha1.ObjectMeta{Name: "gpu-0"}} + b.Notify(WatchEvent{Type: EventTypeModified, Object: gpu}) + } + }(i) + } + + // Concurrent reader + received := 0 + done := make(chan struct{}) + go func() { + for range ch { + received++ + } + close(done) + }() + + wg.Wait() + b.Close() + <-done + + t.Logf("Received %d events (some may be dropped due to buffer)", received) +} + +func TestBroadcaster_Close(t *testing.T) { + logger := klog.Background() + b := NewBroadcaster(logger, 10) + + ch1 := b.Subscribe("sub-1") + ch2 := b.Subscribe("sub-2") + + b.Close() + + if b.SubscriberCount() != 0 { + t.Errorf("Expected 0 subscribers after close, got %d", b.SubscriberCount()) + } + + // Channels should be closed + select { + case _, ok := <-ch1: + if ok { + t.Error("Channel 1 should be closed") + } + default: + t.Error("Channel 1 should be readable (closed)") + } + + select { + case _, ok := <-ch2: + if ok { + t.Error("Channel 2 should be closed") + } + default: + t.Error("Channel 2 should be readable (closed)") + } +} + +func TestBroadcaster_ResubscribeClosesOld(t *testing.T) { + logger := klog.Background() + b := NewBroadcaster(logger, 10) + + // First subscription + ch1 := b.Subscribe("sub-1") + + // Resubscribe with same ID + ch2 := b.Subscribe("sub-1") + + // First channel should be closed + select { + case _, ok := <-ch1: + if ok { + t.Error("First channel should be closed after resubscribe") + } + default: + t.Error("First channel should be readable (closed)") + } + + // Second channel should be active + if b.SubscriberCount() != 1 { + t.Errorf("Expected 1 subscriber, got %d", b.SubscriberCount()) + } + + // Send event - should only go to new channel + gpu := &v1alpha1.Gpu{Metadata: &v1alpha1.ObjectMeta{Name: "gpu-0"}} + b.Notify(WatchEvent{Type: EventTypeAdded, Object: gpu}) + + select { + case <-ch2: + // Good + case <-time.After(time.Second): + t.Error("New channel should receive events") + } +} diff --git a/pkg/deviceapiserver/cache/cache.go b/pkg/deviceapiserver/cache/cache.go new file mode 100644 index 000000000..d909c2ed8 --- /dev/null +++ b/pkg/deviceapiserver/cache/cache.go @@ -0,0 +1,825 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package cache provides a thread-safe cache for GPU resources with +// read-blocking semantics during writes. +// +// The cache uses sync.RWMutex with writer-preference to ensure that +// consumers never read stale data when a provider is updating GPU states. +// When a provider calls an update method, a write lock is acquired which +// blocks all new read operations until the update completes. +package cache + +import ( + "errors" + "strconv" + "sync" + "time" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" + "k8s.io/klog/v2" + + v1alpha1 "github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1" +) + +var ( + // ErrGpuNotFound is returned when a GPU is not found in the cache. + ErrGpuNotFound = errors.New("gpu not found") + + // ErrGpuAlreadyExists is returned when trying to create a GPU that already exists. + ErrGpuAlreadyExists = errors.New("gpu already exists") + + // ErrConflict is returned when a resource version conflict occurs during + // optimistic concurrency control. + ErrConflict = errors.New("resource version conflict") +) + +// Cache operation names for metrics. +const ( + OpCreate = "create" + OpUpdate = "update" + OpUpdateStatus = "update_status" + OpUpdateCondition = "update_condition" + OpDelete = "delete" + OpSet = "set" + // Deprecated: use OpCreate instead. + OpRegister = "register" + // Deprecated: use OpDelete instead. + OpUnregister = "unregister" +) + +// MetricsRecorder is an interface for recording cache operation metrics. +// This allows the cache to record metrics without depending on the metrics package. +type MetricsRecorder interface { + RecordCacheOperation(operation string) +} + +// cachedGpu holds a GPU and its metadata. +type cachedGpu struct { + gpu *v1alpha1.Gpu + resourceVersion int64 + providerID string + lastUpdated time.Time +} + +// GpuCache is a thread-safe cache for GPU resources. +// +// The cache uses sync.RWMutex with writer-preference semantics: +// - Multiple readers can access the cache concurrently +// - When a writer requests the lock, new readers are blocked +// - The writer waits for existing readers to finish, then proceeds +// - Readers blocked during a write resume after the write completes +// +// This ensures consumers never read stale "healthy" data when a provider +// is updating to "unhealthy". +type GpuCache struct { + mu sync.RWMutex + gpus map[string]*cachedGpu + resourceVersion int64 + broadcaster *Broadcaster + logger klog.Logger + metrics MetricsRecorder +} + +// New creates a new GpuCache instance. +// +// The metrics parameter is optional and can be nil if metrics recording +// is not needed (e.g., in tests). +func New(logger klog.Logger, metrics MetricsRecorder) *GpuCache { + return &GpuCache{ + gpus: make(map[string]*cachedGpu), + broadcaster: NewBroadcaster(logger.WithName("broadcaster"), 100), + logger: logger.WithName("cache"), + metrics: metrics, + } +} + +// Broadcaster returns the watch event broadcaster. +func (c *GpuCache) Broadcaster() *Broadcaster { + return c.broadcaster +} + +// recordOperation records a cache operation metric if metrics are configured. +func (c *GpuCache) recordOperation(op string) { + if c.metrics != nil { + c.metrics.RecordCacheOperation(op) + } +} + +// Get retrieves a GPU by name. +// +// This method acquires a read lock and will block if a write is in progress. +// The returned GPU is a deep copy to prevent unintended modifications. +func (c *GpuCache) Get(name string) (*v1alpha1.Gpu, bool) { + c.mu.RLock() + defer c.mu.RUnlock() + + cached, ok := c.gpus[name] + if !ok { + return nil, false + } + + // Return a deep copy to prevent modifications + return proto.Clone(cached.gpu).(*v1alpha1.Gpu), true +} + +// List returns all GPUs in the cache. +// +// This method acquires a read lock and will block if a write is in progress. +// The returned GPUs are deep copies to prevent unintended modifications. +func (c *GpuCache) List() []*v1alpha1.Gpu { + c.mu.RLock() + defer c.mu.RUnlock() + + result := make([]*v1alpha1.Gpu, 0, len(c.gpus)) + + for _, cached := range c.gpus { + result = append(result, proto.Clone(cached.gpu).(*v1alpha1.Gpu)) + } + + return result +} + +// ListWithVersion returns all GPUs and the current resource version atomically +// under a single read lock, ensuring consistency between the list and the version. +func (c *GpuCache) ListWithVersion() ([]*v1alpha1.Gpu, int64) { + c.mu.RLock() + defer c.mu.RUnlock() + + result := make([]*v1alpha1.Gpu, 0, len(c.gpus)) + + for _, cached := range c.gpus { + result = append(result, proto.Clone(cached.gpu).(*v1alpha1.Gpu)) + } + + return result, c.resourceVersion +} + +// ListAndSubscribe atomically lists all GPUs and subscribes to events under +// a single lock, ensuring no events are missed or duplicated between the list +// snapshot and the start of the subscription. +func (c *GpuCache) ListAndSubscribe(subscriberID string) ([]*v1alpha1.Gpu, <-chan WatchEvent) { + c.mu.Lock() + defer c.mu.Unlock() + + // Subscribe while holding the cache lock — any event that occurs after + // this point will be delivered via the channel, not in the list. + events := c.broadcaster.Subscribe(subscriberID) + + result := make([]*v1alpha1.Gpu, 0, len(c.gpus)) + for _, cached := range c.gpus { + result = append(result, proto.Clone(cached.gpu).(*v1alpha1.Gpu)) + } + + return result, events +} + +// Count returns the number of GPUs in the cache. +func (c *GpuCache) Count() int { + c.mu.RLock() + defer c.mu.RUnlock() + + return len(c.gpus) +} + +// ResourceVersion returns the current cache resource version. +// +// This can be used to populate ListMeta.ResourceVersion in list responses. +func (c *GpuCache) ResourceVersion() int64 { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.resourceVersion +} + +// Set creates or updates a GPU in the cache. +// +// If the GPU exists, it is replaced entirely. +// If the GPU doesn't exist, it is created. +// +// This method acquires a write lock, blocking all readers. +// Returns the new resource version. +func (c *GpuCache) Set(gpu *v1alpha1.Gpu) int64 { + c.mu.Lock() + defer c.mu.Unlock() + + c.resourceVersion++ + + // Clone to prevent external modifications + gpuCopy := proto.Clone(gpu).(*v1alpha1.Gpu) + if gpuCopy.Metadata == nil { + gpuCopy.Metadata = &v1alpha1.ObjectMeta{} + } + gpuCopy.Metadata.ResourceVersion = strconv.FormatInt(c.resourceVersion, 10) + + name := gpu.GetMetadata().GetName() + eventType := EventTypeAdded + if _, exists := c.gpus[name]; exists { + eventType = EventTypeModified + } + + c.gpus[name] = &cachedGpu{ + gpu: gpuCopy, + resourceVersion: c.resourceVersion, + lastUpdated: time.Now(), + } + + c.logger.V(1).Info("GPU set", + "name", name, + "resourceVersion", c.resourceVersion, + ) + + // Record metric + c.recordOperation(OpSet) + + // Notify watchers + c.broadcaster.Notify(WatchEvent{ + Type: eventType, + Object: proto.Clone(gpuCopy).(*v1alpha1.Gpu), + }) + + return c.resourceVersion +} + +// Create adds a new GPU to the cache. +// +// If the GPU already exists, returns (nil, ErrGpuAlreadyExists). +// If the GPU is new, returns (gpu, nil) where gpu has the assigned resource_version. +// +// This method acquires a write lock, blocking all readers. +func (c *GpuCache) Create(gpu *v1alpha1.Gpu) (*v1alpha1.Gpu, error) { + c.mu.Lock() + defer c.mu.Unlock() + + name := gpu.GetMetadata().GetName() + + // Check if GPU already exists + if _, ok := c.gpus[name]; ok { + c.logger.V(2).Info("GPU already exists", "name", name) + return nil, ErrGpuAlreadyExists + } + + // Create new GPU + c.resourceVersion++ + + // Clone to prevent external modifications + gpuCopy := proto.Clone(gpu).(*v1alpha1.Gpu) + if gpuCopy.Metadata == nil { + gpuCopy.Metadata = &v1alpha1.ObjectMeta{} + } + gpuCopy.Metadata.ResourceVersion = strconv.FormatInt(c.resourceVersion, 10) + + c.gpus[name] = &cachedGpu{ + gpu: gpuCopy, + resourceVersion: c.resourceVersion, + lastUpdated: time.Now(), + } + + c.logger.V(1).Info("GPU created", + "name", name, + "resourceVersion", c.resourceVersion, + ) + + // Record metric + c.recordOperation(OpCreate) + + // Notify watchers + c.broadcaster.Notify(WatchEvent{ + Type: EventTypeAdded, + Object: proto.Clone(gpuCopy).(*v1alpha1.Gpu), + }) + + return proto.Clone(gpuCopy).(*v1alpha1.Gpu), nil +} + +// Update replaces an existing GPU in the cache. +// +// If expectedVersion is > 0, performs optimistic concurrency check. +// If the GPU doesn't exist, returns (nil, ErrGpuNotFound). +// If version mismatch, returns (nil, ErrConflict). +// +// This method acquires a write lock, blocking all readers. +func (c *GpuCache) Update(gpu *v1alpha1.Gpu, expectedVersion int64) (*v1alpha1.Gpu, error) { + c.mu.Lock() + defer c.mu.Unlock() + + name := gpu.GetMetadata().GetName() + + cached, ok := c.gpus[name] + if !ok { + c.logger.V(2).Info("GPU not found for update", "name", name) + return nil, ErrGpuNotFound + } + + // Check for optimistic concurrency conflict + if expectedVersion > 0 && cached.resourceVersion != expectedVersion { + c.logger.V(2).Info("Resource version conflict", + "name", name, + "expected", expectedVersion, + "actual", cached.resourceVersion, + ) + return nil, ErrConflict + } + + // Update GPU + c.resourceVersion++ + + // Clone to prevent external modifications + gpuCopy := proto.Clone(gpu).(*v1alpha1.Gpu) + if gpuCopy.Metadata == nil { + gpuCopy.Metadata = &v1alpha1.ObjectMeta{} + } + gpuCopy.Metadata.ResourceVersion = strconv.FormatInt(c.resourceVersion, 10) + + c.gpus[name] = &cachedGpu{ + gpu: gpuCopy, + resourceVersion: c.resourceVersion, + providerID: cached.providerID, // Preserve provider ID + lastUpdated: time.Now(), + } + + c.logger.V(1).Info("GPU updated", + "name", name, + "resourceVersion", c.resourceVersion, + ) + + // Record metric + c.recordOperation(OpUpdate) + + // Notify watchers + c.broadcaster.Notify(WatchEvent{ + Type: EventTypeModified, + Object: proto.Clone(gpuCopy).(*v1alpha1.Gpu), + }) + + return proto.Clone(gpuCopy).(*v1alpha1.Gpu), nil +} + +// UpdateStatusWithVersion updates only the status of a GPU with optimistic concurrency. +// +// If expectedVersion is > 0, performs optimistic concurrency check. +// If the GPU doesn't exist, returns (nil, ErrGpuNotFound). +// If version mismatch, returns (nil, ErrConflict). +// +// This method acquires a write lock, blocking ALL readers until complete. +func (c *GpuCache) UpdateStatusWithVersion(name string, status *v1alpha1.GpuStatus, expectedVersion int64) (*v1alpha1.Gpu, error) { + c.mu.Lock() + defer c.mu.Unlock() + + cached, ok := c.gpus[name] + if !ok { + c.logger.V(2).Info("GPU not found for status update", "name", name) + return nil, ErrGpuNotFound + } + + // Check for optimistic concurrency conflict + if expectedVersion > 0 && cached.resourceVersion != expectedVersion { + c.logger.V(2).Info("Resource version conflict", + "name", name, + "expected", expectedVersion, + "actual", cached.resourceVersion, + ) + return nil, ErrConflict + } + + // Update status (clone to isolate cache from caller's pointer) + c.resourceVersion++ + cached.gpu.Status = proto.Clone(status).(*v1alpha1.GpuStatus) + if cached.gpu.Metadata == nil { + cached.gpu.Metadata = &v1alpha1.ObjectMeta{} + } + cached.gpu.Metadata.ResourceVersion = strconv.FormatInt(c.resourceVersion, 10) + cached.resourceVersion = c.resourceVersion + cached.lastUpdated = time.Now() + + c.logger.V(1).Info("GPU status updated", + "name", name, + "resourceVersion", c.resourceVersion, + ) + + // Record metric + c.recordOperation(OpUpdateStatus) + + // Notify watchers + c.broadcaster.Notify(WatchEvent{ + Type: EventTypeModified, + Object: proto.Clone(cached.gpu).(*v1alpha1.Gpu), + }) + + return proto.Clone(cached.gpu).(*v1alpha1.Gpu), nil +} + +// Delete removes a GPU from the cache. +// +// Returns ErrGpuNotFound if the GPU doesn't exist. +// +// This method acquires a write lock, blocking all readers. +func (c *GpuCache) Delete(name string) error { + c.mu.Lock() + defer c.mu.Unlock() + + cached, ok := c.gpus[name] + if !ok { + c.logger.V(2).Info("GPU not found for delete", "name", name) + return ErrGpuNotFound + } + + // Store last known state for watchers + lastState := proto.Clone(cached.gpu).(*v1alpha1.Gpu) + + // Remove from cache + delete(c.gpus, name) + + c.logger.V(1).Info("GPU deleted", + "name", name, + "resourceVersion", cached.resourceVersion, + ) + + // Record metric + c.recordOperation(OpDelete) + + // Notify watchers with last known state + c.broadcaster.Notify(WatchEvent{ + Type: EventTypeDeleted, + Object: lastState, + }) + + return nil +} + +// Register adds a new GPU to the cache. +// +// If the GPU already exists, this returns (false, currentVersion, nil). +// If the GPU is new, this returns (true, newVersion, nil). +// +// This method acquires a write lock, blocking all readers. +func (c *GpuCache) Register( + name string, + spec *v1alpha1.GpuSpec, + initialStatus *v1alpha1.GpuStatus, + providerID string, +) (bool, int64, error) { + c.mu.Lock() + defer c.mu.Unlock() + + // Check if GPU already exists + if cached, ok := c.gpus[name]; ok { + c.logger.V(2).Info("GPU already registered", "name", name, "resourceVersion", cached.resourceVersion) + return false, cached.resourceVersion, nil + } + + // Create new GPU + c.resourceVersion++ + gpu := &v1alpha1.Gpu{ + Metadata: &v1alpha1.ObjectMeta{ + Name: name, + ResourceVersion: strconv.FormatInt(c.resourceVersion, 10), + }, + Spec: spec, + Status: initialStatus, + } + + c.gpus[name] = &cachedGpu{ + gpu: gpu, + resourceVersion: c.resourceVersion, + providerID: providerID, + lastUpdated: time.Now(), + } + + c.logger.V(1).Info("GPU registered", + "name", name, + "providerID", providerID, + "resourceVersion", c.resourceVersion, + ) + + // Record metric + c.recordOperation(OpRegister) + + // Notify watchers + c.broadcaster.Notify(WatchEvent{ + Type: EventTypeAdded, + Object: proto.Clone(gpu).(*v1alpha1.Gpu), + }) + + return true, c.resourceVersion, nil +} + +// Unregister removes a GPU from the cache. +// +// Returns true if the GPU was found and removed, false if not found. +// +// This method acquires a write lock, blocking all readers. +func (c *GpuCache) Unregister(name string) bool { + c.mu.Lock() + defer c.mu.Unlock() + + cached, ok := c.gpus[name] + if !ok { + c.logger.V(2).Info("GPU not found for unregister", "name", name) + return false + } + + // Remove from cache + delete(c.gpus, name) + + c.logger.V(1).Info("GPU unregistered", + "name", name, + "resourceVersion", cached.resourceVersion, + ) + + // Record metric + c.recordOperation(OpUnregister) + + // Notify watchers with last known state + c.broadcaster.Notify(WatchEvent{ + Type: EventTypeDeleted, + Object: proto.Clone(cached.gpu).(*v1alpha1.Gpu), + }) + + return true +} + +// UpdateStatus replaces the entire status of a GPU. +// +// This method acquires a write lock, blocking ALL readers until complete. +// This is the critical path that prevents consumers from reading stale +// "healthy" states when a GPU is transitioning to "unhealthy". +// +// Returns the new resource version, or an error if the GPU is not found. +func (c *GpuCache) UpdateStatus(name string, status *v1alpha1.GpuStatus, providerID string) (int64, error) { + c.mu.Lock() + defer c.mu.Unlock() + + cached, ok := c.gpus[name] + if !ok { + c.logger.V(2).Info("GPU not found for status update", "name", name) + + return 0, ErrGpuNotFound + } + + // Update status (clone to isolate cache from caller's pointer) + c.resourceVersion++ + cached.gpu.Status = proto.Clone(status).(*v1alpha1.GpuStatus) + if cached.gpu.Metadata == nil { + cached.gpu.Metadata = &v1alpha1.ObjectMeta{} + } + cached.gpu.Metadata.ResourceVersion = strconv.FormatInt(c.resourceVersion, 10) + cached.resourceVersion = c.resourceVersion + cached.lastUpdated = time.Now() + + if providerID != "" { + cached.providerID = providerID + } + + c.logger.V(1).Info("GPU status updated", + "name", name, + "providerID", providerID, + "resourceVersion", c.resourceVersion, + ) + + // Record metric + c.recordOperation(OpUpdateStatus) + + // Notify watchers + c.broadcaster.Notify(WatchEvent{ + Type: EventTypeModified, + Object: proto.Clone(cached.gpu).(*v1alpha1.Gpu), + }) + + return c.resourceVersion, nil +} + +// UpdateCondition updates or adds a single condition on a GPU. +// +// If a condition with the same type exists, it is replaced. +// If no condition with that type exists, it is added. +// +// This method acquires a write lock, blocking ALL readers. +// +// Returns the new resource version, or an error if the GPU is not found. +func (c *GpuCache) UpdateCondition(name string, condition *v1alpha1.Condition, providerID string) (int64, error) { + c.mu.Lock() + defer c.mu.Unlock() + + cached, ok := c.gpus[name] + if !ok { + c.logger.V(2).Info("GPU not found for condition update", "name", name) + + return 0, ErrGpuNotFound + } + + // Ensure status exists + if cached.gpu.Status == nil { + cached.gpu.Status = &v1alpha1.GpuStatus{} + } + + // Clone condition to isolate cache from caller's pointer + cond := proto.Clone(condition).(*v1alpha1.Condition) + + // Update last transition time if not set + if cond.LastTransitionTime == nil { + cond.LastTransitionTime = timestamppb.Now() + } + + // Find and update existing condition, or append new one + found := false + + for i, existing := range cached.gpu.Status.Conditions { + if existing.Type == cond.Type { + cached.gpu.Status.Conditions[i] = cond + found = true + + break + } + } + + if !found { + cached.gpu.Status.Conditions = append(cached.gpu.Status.Conditions, cond) + } + + // Update version + c.resourceVersion++ + if cached.gpu.Metadata == nil { + cached.gpu.Metadata = &v1alpha1.ObjectMeta{} + } + cached.gpu.Metadata.ResourceVersion = strconv.FormatInt(c.resourceVersion, 10) + cached.resourceVersion = c.resourceVersion + cached.lastUpdated = time.Now() + + if providerID != "" { + cached.providerID = providerID + } + + c.logger.V(1).Info("GPU condition updated", + "name", name, + "conditionType", condition.Type, + "conditionStatus", condition.Status, + "providerID", providerID, + "resourceVersion", c.resourceVersion, + ) + + // Record metric + c.recordOperation(OpUpdateCondition) + + // Notify watchers + c.broadcaster.Notify(WatchEvent{ + Type: EventTypeModified, + Object: proto.Clone(cached.gpu).(*v1alpha1.Gpu), + }) + + return c.resourceVersion, nil +} + +// Stats returns cache statistics. +type Stats struct { + TotalGpus int + HealthyGpus int + UnhealthyGpus int + UnknownGpus int + ResourceVersion int64 +} + +// MarkProviderGPUsUnknown marks all GPUs registered by a specific provider +// with an Unknown status condition. This is used when a provider's heartbeat +// times out to indicate that GPU health data may be stale. +// +// This method acquires a write lock, blocking all readers. +// Returns the number of GPUs that were marked as Unknown. +func (c *GpuCache) MarkProviderGPUsUnknown(providerID string) int { + var events []WatchEvent + + c.mu.Lock() + count := 0 + for name, cached := range c.gpus { + if cached.providerID != providerID { + continue + } + + // Ensure status exists + if cached.gpu.Status == nil { + cached.gpu.Status = &v1alpha1.GpuStatus{} + } + + // Update or add Ready condition to Unknown + unknownCondition := &v1alpha1.Condition{ + Type: "Ready", + Status: "Unknown", + Message: "Provider heartbeat timeout - GPU health status unknown", + LastTransitionTime: timestamppb.Now(), + } + + found := false + for i, cond := range cached.gpu.Status.Conditions { + if cond.Type == "Ready" { + cached.gpu.Status.Conditions[i] = unknownCondition + found = true + break + } + } + if !found { + cached.gpu.Status.Conditions = append(cached.gpu.Status.Conditions, unknownCondition) + } + + // Update version + c.resourceVersion++ + if cached.gpu.Metadata == nil { + cached.gpu.Metadata = &v1alpha1.ObjectMeta{} + } + cached.gpu.Metadata.ResourceVersion = strconv.FormatInt(c.resourceVersion, 10) + cached.resourceVersion = c.resourceVersion + cached.lastUpdated = time.Now() + + c.logger.Info("Marked GPU as Unknown due to provider heartbeat timeout", + "name", name, + "providerID", providerID, + "resourceVersion", c.resourceVersion, + ) + + events = append(events, WatchEvent{ + Type: EventTypeModified, + Object: proto.Clone(cached.gpu).(*v1alpha1.Gpu), + }) + count++ + } + c.mu.Unlock() + + // Broadcast outside the lock to avoid starving readers + for _, event := range events { + c.broadcaster.Notify(event) + } + + return count +} + +// ListProviderGPUs returns all GPU names registered by a specific provider. +// +// This method acquires a read lock. +func (c *GpuCache) ListProviderGPUs(providerID string) []string { + c.mu.RLock() + defer c.mu.RUnlock() + + var names []string + for name, cached := range c.gpus { + if cached.providerID == providerID { + names = append(names, name) + } + } + return names +} + +// GetStats returns current cache statistics. +func (c *GpuCache) GetStats() Stats { + c.mu.RLock() + defer c.mu.RUnlock() + + stats := Stats{ + TotalGpus: len(c.gpus), + ResourceVersion: c.resourceVersion, + } + + for _, cached := range c.gpus { + if cached.gpu.Status == nil || len(cached.gpu.Status.Conditions) == 0 { + stats.UnknownGpus++ + + continue + } + + // Check Ready condition + classified := false + + for _, cond := range cached.gpu.Status.Conditions { + if cond.Type == "Ready" { + switch cond.Status { + case "True": + stats.HealthyGpus++ + case "False": + stats.UnhealthyGpus++ + default: + stats.UnknownGpus++ + } + + classified = true + + break + } + } + + if !classified { + stats.UnknownGpus++ + } + } + + return stats +} diff --git a/pkg/deviceapiserver/cache/cache_test.go b/pkg/deviceapiserver/cache/cache_test.go new file mode 100644 index 000000000..4e3e478cb --- /dev/null +++ b/pkg/deviceapiserver/cache/cache_test.go @@ -0,0 +1,522 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cache + +import ( + "sync" + "sync/atomic" + "testing" + "time" + + "k8s.io/klog/v2" + + v1alpha1 "github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1" +) + +func TestGpuCache_RegisterAndGet(t *testing.T) { + logger := klog.Background() + c := New(logger, nil) + + // Register a GPU + spec := &v1alpha1.GpuSpec{Uuid: "GPU-1234"} + created, version, err := c.Register("gpu-0", spec, nil, "test-provider") + if err != nil { + t.Fatalf("Register failed: %v", err) + } + if !created { + t.Error("Expected created=true for new GPU") + } + if version != 1 { + t.Errorf("Expected version=1, got %d", version) + } + + // Get the GPU + gpu, found := c.Get("gpu-0") + if !found { + t.Fatal("GPU not found") + } + if gpu.GetMetadata().GetName() != "gpu-0" { + t.Errorf("Expected name=gpu-0, got %s", gpu.GetMetadata().GetName()) + } + if gpu.Spec.Uuid != "GPU-1234" { + t.Errorf("Expected UUID=GPU-1234, got %s", gpu.Spec.Uuid) + } + + // Register same GPU again + created, version, err = c.Register("gpu-0", spec, nil, "test-provider") + if err != nil { + t.Fatalf("Register failed: %v", err) + } + if created { + t.Error("Expected created=false for existing GPU") + } + if version != 1 { + t.Errorf("Expected version=1 (unchanged), got %d", version) + } +} + +func TestGpuCache_Unregister(t *testing.T) { + logger := klog.Background() + c := New(logger, nil) + + // Register a GPU + spec := &v1alpha1.GpuSpec{Uuid: "GPU-1234"} + c.Register("gpu-0", spec, nil, "test-provider") + + // Unregister + deleted := c.Unregister("gpu-0") + if !deleted { + t.Error("Expected deleted=true") + } + + // Verify gone + _, found := c.Get("gpu-0") + if found { + t.Error("GPU should not be found after unregister") + } + + // Unregister again + deleted = c.Unregister("gpu-0") + if deleted { + t.Error("Expected deleted=false for non-existent GPU") + } +} + +func TestGpuCache_UpdateStatus(t *testing.T) { + logger := klog.Background() + c := New(logger, nil) + + // Register a GPU + spec := &v1alpha1.GpuSpec{Uuid: "GPU-1234"} + c.Register("gpu-0", spec, nil, "test-provider") + + // Update status + status := &v1alpha1.GpuStatus{ + Conditions: []*v1alpha1.Condition{ + {Type: "Ready", Status: "True"}, + }, + } + version, err := c.UpdateStatus("gpu-0", status, "test-provider") + if err != nil { + t.Fatalf("UpdateStatus failed: %v", err) + } + if version != 2 { + t.Errorf("Expected version=2, got %d", version) + } + + // Verify status + gpu, _ := c.Get("gpu-0") + if len(gpu.Status.Conditions) != 1 { + t.Errorf("Expected 1 condition, got %d", len(gpu.Status.Conditions)) + } + if gpu.Status.Conditions[0].Status != "True" { + t.Errorf("Expected status=True, got %s", gpu.Status.Conditions[0].Status) + } + + // Update non-existent GPU + _, err = c.UpdateStatus("gpu-999", status, "test-provider") + if err != ErrGpuNotFound { + t.Errorf("Expected ErrGpuNotFound, got %v", err) + } +} + +func TestGpuCache_UpdateCondition(t *testing.T) { + logger := klog.Background() + c := New(logger, nil) + + // Register a GPU with initial status + spec := &v1alpha1.GpuSpec{Uuid: "GPU-1234"} + initialStatus := &v1alpha1.GpuStatus{ + Conditions: []*v1alpha1.Condition{ + {Type: "Ready", Status: "True"}, + }, + } + c.Register("gpu-0", spec, initialStatus, "test-provider") + + // Add new condition + condition := &v1alpha1.Condition{Type: "Healthy", Status: "True"} + version, err := c.UpdateCondition("gpu-0", condition, "health-monitor") + if err != nil { + t.Fatalf("UpdateCondition failed: %v", err) + } + if version != 2 { + t.Errorf("Expected version=2, got %d", version) + } + + // Verify both conditions exist + gpu, _ := c.Get("gpu-0") + if len(gpu.Status.Conditions) != 2 { + t.Errorf("Expected 2 conditions, got %d", len(gpu.Status.Conditions)) + } + + // Update existing condition + condition = &v1alpha1.Condition{Type: "Ready", Status: "False"} + version, err = c.UpdateCondition("gpu-0", condition, "test-provider") + if err != nil { + t.Fatalf("UpdateCondition failed: %v", err) + } + if version != 3 { + t.Errorf("Expected version=3, got %d", version) + } + + // Verify condition was updated, not added + gpu, _ = c.Get("gpu-0") + if len(gpu.Status.Conditions) != 2 { + t.Errorf("Expected 2 conditions (updated, not added), got %d", len(gpu.Status.Conditions)) + } +} + +func TestGpuCache_List(t *testing.T) { + logger := klog.Background() + c := New(logger, nil) + + // Empty list + gpus := c.List() + if len(gpus) != 0 { + t.Errorf("Expected empty list, got %d", len(gpus)) + } + + // Add GPUs + c.Register("gpu-0", &v1alpha1.GpuSpec{Uuid: "GPU-0"}, nil, "p") + c.Register("gpu-1", &v1alpha1.GpuSpec{Uuid: "GPU-1"}, nil, "p") + c.Register("gpu-2", &v1alpha1.GpuSpec{Uuid: "GPU-2"}, nil, "p") + + gpus = c.List() + if len(gpus) != 3 { + t.Errorf("Expected 3 GPUs, got %d", len(gpus)) + } +} + +func TestGpuCache_GetStats(t *testing.T) { + logger := klog.Background() + c := New(logger, nil) + + // Add GPUs with various states + c.Register("gpu-0", &v1alpha1.GpuSpec{Uuid: "GPU-0"}, &v1alpha1.GpuStatus{ + Conditions: []*v1alpha1.Condition{{Type: "Ready", Status: "True"}}, + }, "p") + c.Register("gpu-1", &v1alpha1.GpuSpec{Uuid: "GPU-1"}, &v1alpha1.GpuStatus{ + Conditions: []*v1alpha1.Condition{{Type: "Ready", Status: "False"}}, + }, "p") + c.Register("gpu-2", &v1alpha1.GpuSpec{Uuid: "GPU-2"}, nil, "p") // No status + + stats := c.GetStats() + if stats.TotalGpus != 3 { + t.Errorf("Expected TotalGpus=3, got %d", stats.TotalGpus) + } + if stats.HealthyGpus != 1 { + t.Errorf("Expected HealthyGpus=1, got %d", stats.HealthyGpus) + } + if stats.UnhealthyGpus != 1 { + t.Errorf("Expected UnhealthyGpus=1, got %d", stats.UnhealthyGpus) + } + if stats.UnknownGpus != 1 { + t.Errorf("Expected UnknownGpus=1, got %d", stats.UnknownGpus) + } +} + +// TestGpuCache_ReadBlocksDuringWrite verifies the critical read-blocking behavior. +// When a write is in progress, new readers MUST block until the write completes. +func TestGpuCache_ReadBlocksDuringWrite(t *testing.T) { + logger := klog.Background() + c := New(logger, nil) + + // Register a GPU + c.Register("gpu-0", &v1alpha1.GpuSpec{Uuid: "GPU-0"}, nil, "p") + + var ( + writeStarted = make(chan struct{}) + writeComplete = make(chan struct{}) + readStarted = make(chan struct{}) + readComplete = make(chan struct{}) + readBlocked atomic.Bool + ) + + // Start a slow write that holds the lock + go func() { + c.mu.Lock() + close(writeStarted) + // Hold the lock for 100ms + time.Sleep(100 * time.Millisecond) + c.mu.Unlock() + close(writeComplete) + }() + + // Wait for write to acquire lock + <-writeStarted + + // Start a read - should block + go func() { + close(readStarted) + _, _ = c.Get("gpu-0") // This should block + close(readComplete) + }() + + <-readStarted + // Give the read goroutine time to attempt the lock + time.Sleep(20 * time.Millisecond) + + // Check if read has completed (it shouldn't have) + select { + case <-readComplete: + t.Fatal("Read completed while write lock was held - blocking failed!") + default: + readBlocked.Store(true) + } + + // Wait for write to complete + <-writeComplete + + // Now read should complete + select { + case <-readComplete: + // Expected - read completed after write released + case <-time.After(200 * time.Millisecond): + t.Fatal("Read did not complete after write released") + } + + if !readBlocked.Load() { + t.Error("Read was not blocked during write") + } +} + +// TestGpuCache_ConcurrentReads verifies multiple readers can access concurrently. +func TestGpuCache_ConcurrentReads(t *testing.T) { + logger := klog.Background() + c := New(logger, nil) + + // Register some GPUs + for i := 0; i < 10; i++ { + c.Register( + "gpu-"+string(rune('0'+i)), + &v1alpha1.GpuSpec{Uuid: "GPU"}, + nil, + "p", + ) + } + + // Start many concurrent readers + var wg sync.WaitGroup + errors := make(chan error, 100) + + for i := 0; i < 100; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 100; j++ { + gpus := c.List() + if len(gpus) != 10 { + errors <- nil // Signal unexpected count + } + } + }() + } + + wg.Wait() + close(errors) + + for range errors { + t.Error("Concurrent read returned unexpected result") + } +} + +func TestGpuCache_MarkProviderGPUsUnknown(t *testing.T) { + logger := klog.Background() + c := New(logger, nil) + + // Register GPUs from different providers + c.Register("gpu-0", &v1alpha1.GpuSpec{Uuid: "GPU-0"}, &v1alpha1.GpuStatus{ + Conditions: []*v1alpha1.Condition{{Type: "Ready", Status: "True"}}, + }, "provider-a") + c.Register("gpu-1", &v1alpha1.GpuSpec{Uuid: "GPU-1"}, &v1alpha1.GpuStatus{ + Conditions: []*v1alpha1.Condition{{Type: "Ready", Status: "True"}}, + }, "provider-a") + c.Register("gpu-2", &v1alpha1.GpuSpec{Uuid: "GPU-2"}, &v1alpha1.GpuStatus{ + Conditions: []*v1alpha1.Condition{{Type: "Ready", Status: "True"}}, + }, "provider-b") + + // Mark provider-a's GPUs as Unknown + count := c.MarkProviderGPUsUnknown("provider-a") + if count != 2 { + t.Errorf("Expected 2 GPUs marked, got %d", count) + } + + // Verify provider-a's GPUs are now Unknown + gpu0, _ := c.Get("gpu-0") + var gpu0Ready *v1alpha1.Condition + for _, cond := range gpu0.Status.Conditions { + if cond.Type == "Ready" { + gpu0Ready = cond + break + } + } + if gpu0Ready == nil || gpu0Ready.Status != "Unknown" { + t.Errorf("Expected gpu-0 Ready=Unknown, got %v", gpu0Ready) + } + + gpu1, _ := c.Get("gpu-1") + var gpu1Ready *v1alpha1.Condition + for _, cond := range gpu1.Status.Conditions { + if cond.Type == "Ready" { + gpu1Ready = cond + break + } + } + if gpu1Ready == nil || gpu1Ready.Status != "Unknown" { + t.Errorf("Expected gpu-1 Ready=Unknown, got %v", gpu1Ready) + } + + // Verify provider-b's GPU is still healthy + gpu2, _ := c.Get("gpu-2") + var gpu2Ready *v1alpha1.Condition + for _, cond := range gpu2.Status.Conditions { + if cond.Type == "Ready" { + gpu2Ready = cond + break + } + } + if gpu2Ready == nil || gpu2Ready.Status != "True" { + t.Errorf("Expected gpu-2 Ready=True (unchanged), got %v", gpu2Ready) + } + + // Mark non-existent provider (should return 0) + count = c.MarkProviderGPUsUnknown("provider-c") + if count != 0 { + t.Errorf("Expected 0 GPUs marked for non-existent provider, got %d", count) + } +} + +func TestGpuCache_ListProviderGPUs(t *testing.T) { + logger := klog.Background() + c := New(logger, nil) + + // Register GPUs from different providers + c.Register("gpu-0", &v1alpha1.GpuSpec{Uuid: "GPU-0"}, nil, "provider-a") + c.Register("gpu-1", &v1alpha1.GpuSpec{Uuid: "GPU-1"}, nil, "provider-a") + c.Register("gpu-2", &v1alpha1.GpuSpec{Uuid: "GPU-2"}, nil, "provider-b") + + // List provider-a's GPUs + names := c.ListProviderGPUs("provider-a") + if len(names) != 2 { + t.Errorf("Expected 2 GPUs for provider-a, got %d", len(names)) + } + + // List provider-b's GPUs + names = c.ListProviderGPUs("provider-b") + if len(names) != 1 { + t.Errorf("Expected 1 GPU for provider-b, got %d", len(names)) + } + + // List non-existent provider's GPUs + names = c.ListProviderGPUs("provider-c") + if len(names) != 0 { + t.Errorf("Expected 0 GPUs for non-existent provider, got %d", len(names)) + } +} + +// TestGpuCache_WriteBlocksNewReaders verifies writer-preference behavior. +// When a write is pending, new readers should block (not cut in line). +func TestGpuCache_WriteBlocksNewReaders(t *testing.T) { + logger := klog.Background() + c := New(logger, nil) + + c.Register("gpu-0", &v1alpha1.GpuSpec{Uuid: "GPU-0"}, nil, "p") + + var ( + firstReadDone = make(chan struct{}) + writePending = make(chan struct{}) + secondReadStart = make(chan struct{}) + order []string + orderMu sync.Mutex + ) + + // First reader - gets the lock + go func() { + c.mu.RLock() + orderMu.Lock() + order = append(order, "read1-start") + orderMu.Unlock() + + // Wait for write to be pending + <-writePending + time.Sleep(50 * time.Millisecond) + + orderMu.Lock() + order = append(order, "read1-end") + orderMu.Unlock() + c.mu.RUnlock() + close(firstReadDone) + }() + + // Give first reader time to acquire lock + time.Sleep(10 * time.Millisecond) + + // Writer - will wait for first reader, then block second reader + go func() { + close(writePending) + c.mu.Lock() + orderMu.Lock() + order = append(order, "write") + orderMu.Unlock() + c.mu.Unlock() + }() + + // Second reader - should wait for writer + go func() { + <-secondReadStart + c.mu.RLock() + orderMu.Lock() + order = append(order, "read2") + orderMu.Unlock() + c.mu.RUnlock() + }() + + // Start second reader after write is pending + time.Sleep(30 * time.Millisecond) + close(secondReadStart) + + // Wait for everything to complete + <-firstReadDone + time.Sleep(100 * time.Millisecond) + + orderMu.Lock() + defer orderMu.Unlock() + + // Expected order: read1-start, read1-end, write, read2 + // Writer should execute before second reader due to writer-preference + if len(order) < 4 { + t.Fatalf("Not all operations completed: %v", order) + } + + // The write should come before read2 + writeIdx := -1 + read2Idx := -1 + for i, op := range order { + if op == "write" { + writeIdx = i + } + if op == "read2" { + read2Idx = i + } + } + + if writeIdx == -1 || read2Idx == -1 { + t.Fatalf("Missing operations: %v", order) + } + + if writeIdx > read2Idx { + t.Errorf("Writer should execute before second reader (writer-preference). Order: %v", order) + } +} diff --git a/pkg/deviceapiserver/config.go b/pkg/deviceapiserver/config.go new file mode 100644 index 000000000..beb793e73 --- /dev/null +++ b/pkg/deviceapiserver/config.go @@ -0,0 +1,229 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deviceapiserver + +import ( + "errors" + "flag" + "fmt" + "os" + "strconv" + "time" +) + +// Config holds the server configuration. +type Config struct { + // GRPCAddress is the TCP address for gRPC (e.g., ":50051"). + // Default is "127.0.0.1:50051" (localhost only) for security. + // Set to ":50051" to bind to all interfaces (use with caution - unauthenticated). + GRPCAddress string + + // ProviderAddress is the TCP address for provider gRPC connections (e.g., "localhost:9001"). + // This is used by containerized providers (sidecars) to connect to the server. + // If empty, providers must use the main GRPCAddress or UnixSocket. + // Default is "localhost:9001" when a separate provider endpoint is needed. + ProviderAddress string + + // UnixSocket is the path to the Unix socket for node-local communication. + // If empty, Unix socket is disabled. + UnixSocket string + + // UnixSocketPermissions is the file mode for the Unix socket. + UnixSocketPermissions os.FileMode + + // HealthPort is the port for HTTP health endpoints. + HealthPort int + + // MetricsPort is the port for Prometheus metrics. + MetricsPort int + + // ShutdownTimeout is the maximum time to wait for graceful shutdown. + ShutdownTimeout time.Duration + + // ShutdownDelay is the time to wait before starting shutdown (for k8s readiness). + ShutdownDelay time.Duration + + // LogFormat is the log output format ("text" or "json"). + LogFormat string + + // NodeName is the Kubernetes node name (from downward API). + NodeName string + + // NVML Provider configuration + // NVMLEnabled enables the built-in NVML provider for device enumeration and health monitoring. + NVMLEnabled bool + + // NVMLDriverRoot is the root path where NVIDIA driver libraries are located. + // Common values: "/run/nvidia/driver" (container with RuntimeClass), "/" (bare metal) + NVMLDriverRoot string + + // NVMLIgnoredXids is a comma-separated list of additional XID error codes to ignore. + NVMLIgnoredXids string + + // NVMLHealthCheckEnabled enables XID event monitoring for health checks. + NVMLHealthCheckEnabled bool +} + +// DefaultConfig returns a Config with default values. +func DefaultConfig() Config { + return Config{ + // Bind to localhost only by default for security (unauthenticated API). + // Use ":50051" to bind to all interfaces if network access is required. + GRPCAddress: "127.0.0.1:50051", + // Provider address is empty by default (disabled). + // Set to "localhost:9001" when using containerized provider sidecars. + ProviderAddress: "", + UnixSocket: "/var/run/device-api/device.sock", + UnixSocketPermissions: 0660, + HealthPort: 8081, + MetricsPort: 9090, + ShutdownTimeout: 30 * time.Second, + ShutdownDelay: 5 * time.Second, + LogFormat: "text", + NodeName: os.Getenv("NODE_NAME"), + + // NVML defaults - disabled by default for safety + NVMLEnabled: false, + NVMLDriverRoot: "/run/nvidia/driver", + NVMLIgnoredXids: "", + NVMLHealthCheckEnabled: true, + } +} + +// BindFlags binds configuration flags to the given flag set. +func (c *Config) BindFlags(fs *flag.FlagSet) { + fs.StringVar(&c.GRPCAddress, "grpc-address", c.GRPCAddress, + "TCP address for gRPC server (e.g., :50051 for all interfaces). "+ + "WARNING: The gRPC API is unauthenticated. Default binds to localhost only.") + fs.StringVar(&c.ProviderAddress, "provider-address", c.ProviderAddress, + "TCP address for provider gRPC connections (e.g., localhost:9001). "+ + "Used by containerized provider sidecars. Empty to disable.") + fs.StringVar(&c.UnixSocket, "unix-socket", c.UnixSocket, + "Path to Unix socket for node-local IPC (empty to disable)") + fs.IntVar(&c.HealthPort, "health-port", c.HealthPort, + "Port for HTTP health endpoints (/healthz, /readyz)") + fs.IntVar(&c.MetricsPort, "metrics-port", c.MetricsPort, + "Port for Prometheus metrics (/metrics)") + + // Duration flags need special handling + shutdownTimeout := int(c.ShutdownTimeout.Seconds()) + fs.IntVar(&shutdownTimeout, "shutdown-timeout", shutdownTimeout, + "Maximum time in seconds to wait for graceful shutdown") + + shutdownDelay := int(c.ShutdownDelay.Seconds()) + fs.IntVar(&shutdownDelay, "shutdown-delay", shutdownDelay, + "Time in seconds to wait before starting shutdown (for k8s readiness propagation)") + + fs.StringVar(&c.LogFormat, "log-format", c.LogFormat, + "Log output format: text or json") + fs.StringVar(&c.NodeName, "node-name", c.NodeName, + "Kubernetes node name (defaults to NODE_NAME env var)") +} + +// ApplyEnvironment overrides config from environment variables. +func (c *Config) ApplyEnvironment() { + c.applyServerEnv() + c.applyNVMLEnv() +} + +// applyServerEnv applies server-related environment variables. +func (c *Config) applyServerEnv() { + if v := os.Getenv("DEVICE_API_GRPC_ADDRESS"); v != "" { + c.GRPCAddress = v + } + + if v := os.Getenv("DEVICE_API_PROVIDER_ADDRESS"); v != "" { + c.ProviderAddress = v + } + + if v := os.Getenv("DEVICE_API_UNIX_SOCKET"); v != "" { + c.UnixSocket = v + } + + if v := os.Getenv("DEVICE_API_HEALTH_PORT"); v != "" { + if port, err := strconv.Atoi(v); err == nil { + c.HealthPort = port + } + } + + if v := os.Getenv("DEVICE_API_METRICS_PORT"); v != "" { + if port, err := strconv.Atoi(v); err == nil { + c.MetricsPort = port + } + } + + if v := os.Getenv("DEVICE_API_LOG_FORMAT"); v != "" { + c.LogFormat = v + } + + if v := os.Getenv("NODE_NAME"); v != "" && c.NodeName == "" { + c.NodeName = v + } +} + +// applyNVMLEnv applies NVML-related environment variables. +func (c *Config) applyNVMLEnv() { + if v := os.Getenv("DEVICE_API_NVML_ENABLED"); v == "true" || v == "1" { + c.NVMLEnabled = true + } + + if v := os.Getenv("DEVICE_API_NVML_DRIVER_ROOT"); v != "" { + c.NVMLDriverRoot = v + } + + if v := os.Getenv("DEVICE_API_NVML_IGNORED_XIDS"); v != "" { + c.NVMLIgnoredXids = v + } + + if v := os.Getenv("DEVICE_API_NVML_HEALTH_CHECK"); v == "false" || v == "0" { + c.NVMLHealthCheckEnabled = false + } +} + +// Validate checks the configuration for errors. +func (c *Config) Validate() error { + var errs []error + + if c.GRPCAddress == "" && c.UnixSocket == "" { + errs = append(errs, errors.New("at least one of grpc-address or unix-socket must be specified")) + } + + if err := validatePort("health-port", c.HealthPort); err != nil { + errs = append(errs, err) + } + + if err := validatePort("metrics-port", c.MetricsPort); err != nil { + errs = append(errs, err) + } + + if c.ShutdownTimeout < 0 { + errs = append(errs, errors.New("shutdown-timeout must be non-negative")) + } + + if c.LogFormat != "text" && c.LogFormat != "json" { + errs = append(errs, fmt.Errorf("log-format must be 'text' or 'json', got %q", c.LogFormat)) + } + + return errors.Join(errs...) +} + +// validatePort checks if a port number is valid. +func validatePort(name string, port int) error { + if port < 0 || port > 65535 { + return fmt.Errorf("%s must be 0-65535, got %d", name, port) + } + + return nil +} diff --git a/pkg/deviceapiserver/crdpublisher/metrics.go b/pkg/deviceapiserver/crdpublisher/metrics.go new file mode 100644 index 000000000..2f89fc78d --- /dev/null +++ b/pkg/deviceapiserver/crdpublisher/metrics.go @@ -0,0 +1,55 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crdpublisher + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + // healthEventsCreatedTotal tracks the number of HealthEvent CRDs created. + healthEventsCreatedTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "nvsentinel", + Subsystem: "crd_publisher", + Name: "health_events_created_total", + Help: "Total number of HealthEvent CRDs created", + }, + []string{"node", "source"}, + ) + + // healthEventsFailedTotal tracks failed CRD creations. + healthEventsFailedTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "nvsentinel", + Subsystem: "crd_publisher", + Name: "health_events_failed_total", + Help: "Total number of failed HealthEvent CRD creations", + }, + []string{"node", "source", "reason"}, + ) + + // debouncedEventsTotal tracks events that were debounced. + debouncedEventsTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "nvsentinel", + Subsystem: "crd_publisher", + Name: "debounced_events_total", + Help: "Total number of events debounced", + }, + []string{"node"}, + ) +) diff --git a/pkg/deviceapiserver/crdpublisher/publisher.go b/pkg/deviceapiserver/crdpublisher/publisher.go new file mode 100644 index 000000000..84806ebd4 --- /dev/null +++ b/pkg/deviceapiserver/crdpublisher/publisher.go @@ -0,0 +1,334 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package crdpublisher creates HealthEvent CRDs from GPU status changes. +package crdpublisher + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + devicev1alpha1 "github.com/nvidia/nvsentinel/api/device/v1alpha1" + nvsentinelv1alpha1 "github.com/nvidia/nvsentinel/api/nvsentinel/v1alpha1" +) + +// Config configures the CRD publisher. +type Config struct { + // NodeName is the name of the node this publisher runs on. + NodeName string + + // Enabled controls whether CRD publishing is active. + Enabled bool + + // DebounceInterval is the minimum time between CRD updates for the same GPU. + DebounceInterval time.Duration + + // Source identifies this publisher (e.g., "nvml-health-monitor"). + Source string +} + +// Publisher creates HealthEvent CRDs when GPU status changes. +type Publisher struct { + config Config + client client.Client + logger klog.Logger + + // Long-lived context for background goroutines, set by Start() + ctx context.Context + + // Debouncing + mu sync.Mutex + lastPublish map[string]time.Time // GPU UUID -> last publish time + pendingGPUs map[string]*devicev1alpha1.GPU + debounceStop chan struct{} + stopOnce sync.Once +} + +// New creates a new CRD publisher. +func New(config Config, logger klog.Logger) (*Publisher, error) { + if !config.Enabled { + return &Publisher{config: config, logger: logger}, nil + } + + // Create in-cluster client + restConfig, err := rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("failed to get in-cluster config: %w", err) + } + + // Register nvsentinel scheme + if err := nvsentinelv1alpha1.AddToScheme(scheme.Scheme); err != nil { + return nil, fmt.Errorf("failed to register nvsentinel scheme: %w", err) + } + + k8sClient, err := client.New(restConfig, client.Options{Scheme: scheme.Scheme}) + if err != nil { + return nil, fmt.Errorf("failed to create k8s client: %w", err) + } + + if config.DebounceInterval == 0 { + config.DebounceInterval = 5 * time.Second + } + if config.Source == "" { + config.Source = "device-api-server" + } + + return &Publisher{ + config: config, + client: k8sClient, + logger: logger.WithName("crd-publisher"), + lastPublish: make(map[string]time.Time), + pendingGPUs: make(map[string]*devicev1alpha1.GPU), + debounceStop: make(chan struct{}), + }, nil +} + +// Start starts the debounce processor. +func (p *Publisher) Start(ctx context.Context) { + if !p.config.Enabled { + return + } + + p.ctx = ctx + go p.runDebounceProcessor(ctx) +} + +// Stop stops the publisher. Safe to call multiple times. +func (p *Publisher) Stop() { + p.stopOnce.Do(func() { + if p.debounceStop != nil { + close(p.debounceStop) + } + }) +} + +// OnGPUUnhealthy is called when a GPU becomes unhealthy. +// It creates or updates a HealthEvent CRD. +func (p *Publisher) OnGPUUnhealthy(ctx context.Context, gpu *devicev1alpha1.GPU) { + if !p.config.Enabled || p.client == nil { + return + } + + p.mu.Lock() + defer p.mu.Unlock() + + uuid := gpu.Spec.UUID + if uuid == "" { + p.logger.V(1).Info("GPU has no UUID, skipping CRD publish") + return + } + + // Check debounce + if lastTime, ok := p.lastPublish[uuid]; ok { + if time.Since(lastTime) < p.config.DebounceInterval { + // Queue for later + p.pendingGPUs[uuid] = gpu + p.logger.V(2).Info("Debouncing GPU update", "uuid", uuid) + return + } + } + + // Publish immediately using the long-lived context from Start(), + // not the request-scoped ctx, since the goroutine may outlive the request. + p.lastPublish[uuid] = time.Now() + publishCtx := p.ctx + if publishCtx == nil { + publishCtx = ctx + } + go p.publishHealthEvent(publishCtx, gpu) +} + +// OnGPUHealthy is called when a GPU becomes healthy again. +// It can update an existing HealthEvent CRD to resolved status. +func (p *Publisher) OnGPUHealthy(ctx context.Context, gpu *devicev1alpha1.GPU) { + if !p.config.Enabled || p.client == nil { + return + } + + // For now, we don't auto-resolve events + // The resolution should come from the remediation workflow + p.logger.V(1).Info("GPU healthy, consider manual resolution", + "uuid", gpu.Spec.UUID, + "nodeName", p.config.NodeName, + ) +} + +// runDebounceProcessor processes pending GPU updates. +func (p *Publisher) runDebounceProcessor(ctx context.Context) { + ticker := time.NewTicker(p.config.DebounceInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-p.debounceStop: + return + case <-ticker.C: + p.processPendingGPUs(ctx) + } + } +} + +// processPendingGPUs publishes any pending GPU updates. +func (p *Publisher) processPendingGPUs(ctx context.Context) { + p.mu.Lock() + pending := make(map[string]*devicev1alpha1.GPU) + for uuid, gpu := range p.pendingGPUs { + pending[uuid] = gpu + } + p.pendingGPUs = make(map[string]*devicev1alpha1.GPU) + p.mu.Unlock() + + for uuid, gpu := range pending { + p.mu.Lock() + p.lastPublish[uuid] = time.Now() + p.mu.Unlock() + + p.publishHealthEvent(ctx, gpu) + } +} + +// publishHealthEvent creates or updates a HealthEvent CRD. +func (p *Publisher) publishHealthEvent(ctx context.Context, gpu *devicev1alpha1.GPU) { + log := p.logger.WithValues("uuid", gpu.Spec.UUID) + + // Extract error info from GPU status + errorCodes, message, isFatal := p.extractErrorInfo(gpu) + + // Generate event name: nodename-gpuindex-errorcode-timestamp + eventName := p.generateEventName(gpu) + + healthEvent := &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: eventName, + Labels: map[string]string{ + "nvsentinel.nvidia.com/node": p.config.NodeName, + "nvsentinel.nvidia.com/component-class": "gpu", + "nvsentinel.nvidia.com/is-fatal": fmt.Sprintf("%t", isFatal), + "nvsentinel.nvidia.com/source": p.config.Source, + }, + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: p.config.Source, + NodeName: p.config.NodeName, + ComponentClass: "GPU", + CheckName: p.determineCheckName(gpu), + IsFatal: isFatal, + IsHealthy: false, + Message: message, + ErrorCodes: errorCodes, + DetectedAt: metav1.Now(), + EntitiesImpacted: []nvsentinelv1alpha1.Entity{ + {Type: "GPU", Value: gpu.Spec.UUID}, + }, + Metadata: map[string]string{ + "uuid": gpu.Spec.UUID, + }, + }, + Status: nvsentinelv1alpha1.HealthEventStatus{ + Phase: nvsentinelv1alpha1.PhaseNew, + }, + } + + // Try to create the HealthEvent + err := p.client.Create(ctx, healthEvent) + if err != nil { + if apierrors.IsAlreadyExists(err) { + log.V(1).Info("HealthEvent already exists", "name", eventName) + return + } + log.Error(err, "Failed to create HealthEvent", "name", eventName) + return + } + + log.Info("Created HealthEvent CRD", + "name", eventName, + "isFatal", isFatal, + "errorCodes", errorCodes, + ) + + // Record metric + healthEventsCreatedTotal.WithLabelValues(p.config.NodeName, p.config.Source).Inc() +} + +// extractErrorInfo extracts error information from GPU status conditions. +func (p *Publisher) extractErrorInfo(gpu *devicev1alpha1.GPU) (errorCodes []string, message string, isFatal bool) { + for _, cond := range gpu.Status.Conditions { + if cond.Type == "Healthy" && cond.Status == metav1.ConditionFalse { + message = cond.Message + if cond.Reason != "" { + // Extract XID from reason if present (e.g., "XID79") + if strings.HasPrefix(cond.Reason, "XID") { + xid := strings.TrimPrefix(cond.Reason, "XID") + errorCodes = append(errorCodes, xid) + } + } + } + } + + // Determine if fatal based on recommended action + isFatal = gpu.Status.RecommendedAction == "RESTART_BM" || + gpu.Status.RecommendedAction == "RESTART_VM" || + gpu.Status.RecommendedAction == "REPLACE_VM" + + if message == "" { + message = "GPU reported unhealthy" + } + + return errorCodes, message, isFatal +} + +// generateEventName creates a unique name for the HealthEvent. +func (p *Publisher) generateEventName(gpu *devicev1alpha1.GPU) string { + // Use UUID short form + timestamp + uuid := gpu.Spec.UUID + if len(uuid) > 12 { + uuid = uuid[len(uuid)-12:] + } + uuid = strings.ReplaceAll(uuid, "-", "") + uuid = strings.ToLower(uuid) + + timestamp := time.Now().UnixNano() + + return fmt.Sprintf("%s-%s-%d", p.config.NodeName, uuid, timestamp) +} + +// determineCheckName determines the check name based on GPU conditions. +func (p *Publisher) determineCheckName(gpu *devicev1alpha1.GPU) string { + for _, cond := range gpu.Status.Conditions { + if cond.Type == "Healthy" && cond.Status == metav1.ConditionFalse { + if strings.Contains(cond.Reason, "XID") { + return "xid-error-check" + } + if strings.Contains(cond.Reason, "ECC") { + return "memory-ecc-check" + } + if strings.Contains(cond.Reason, "Thermal") || strings.Contains(cond.Reason, "Temperature") { + return "thermal-check" + } + } + } + return "health-check" +} diff --git a/pkg/deviceapiserver/crdpublisher/publisher_test.go b/pkg/deviceapiserver/crdpublisher/publisher_test.go new file mode 100644 index 000000000..4efd2e443 --- /dev/null +++ b/pkg/deviceapiserver/crdpublisher/publisher_test.go @@ -0,0 +1,337 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crdpublisher + +import ( + "context" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + devicev1alpha1 "github.com/nvidia/nvsentinel/api/device/v1alpha1" + nvsentinelv1alpha1 "github.com/nvidia/nvsentinel/api/nvsentinel/v1alpha1" +) + +func TestPublisher_ExtractErrorInfo(t *testing.T) { + p := &Publisher{ + config: Config{NodeName: "test-node"}, + } + + tests := []struct { + name string + gpu *devicev1alpha1.GPU + wantErrorCodes []string + wantMessage string + wantFatal bool + }{ + { + name: "XID error should be fatal", + gpu: &devicev1alpha1.GPU{ + Status: devicev1alpha1.GPUStatus{ + Conditions: []metav1.Condition{ + { + Type: "Healthy", + Status: metav1.ConditionFalse, + Reason: "XID79", + Message: "GPU has fallen off the bus", + }, + }, + RecommendedAction: "RESTART_BM", + }, + }, + wantErrorCodes: []string{"79"}, + wantMessage: "GPU has fallen off the bus", + wantFatal: true, + }, + { + name: "non-fatal error", + gpu: &devicev1alpha1.GPU{ + Status: devicev1alpha1.GPUStatus{ + Conditions: []metav1.Condition{ + { + Type: "Healthy", + Status: metav1.ConditionFalse, + Reason: "HighTemperature", + Message: "GPU temperature above threshold", + }, + }, + RecommendedAction: "COMPONENT_RESET", + }, + }, + wantErrorCodes: nil, + wantMessage: "GPU temperature above threshold", + wantFatal: false, + }, + { + name: "healthy GPU", + gpu: &devicev1alpha1.GPU{ + Status: devicev1alpha1.GPUStatus{ + Conditions: []metav1.Condition{ + { + Type: "Healthy", + Status: metav1.ConditionTrue, + }, + }, + }, + }, + wantErrorCodes: nil, + wantMessage: "GPU reported unhealthy", + wantFatal: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + errorCodes, message, isFatal := p.extractErrorInfo(tt.gpu) + + if len(errorCodes) != len(tt.wantErrorCodes) { + t.Errorf("extractErrorInfo() errorCodes = %v, want %v", errorCodes, tt.wantErrorCodes) + } + for i, code := range errorCodes { + if code != tt.wantErrorCodes[i] { + t.Errorf("extractErrorInfo() errorCodes[%d] = %v, want %v", i, code, tt.wantErrorCodes[i]) + } + } + + if message != tt.wantMessage { + t.Errorf("extractErrorInfo() message = %v, want %v", message, tt.wantMessage) + } + + if isFatal != tt.wantFatal { + t.Errorf("extractErrorInfo() isFatal = %v, want %v", isFatal, tt.wantFatal) + } + }) + } +} + +func TestPublisher_DetermineCheckName(t *testing.T) { + p := &Publisher{} + + tests := []struct { + name string + gpu *devicev1alpha1.GPU + want string + }{ + { + name: "XID error", + gpu: &devicev1alpha1.GPU{ + Status: devicev1alpha1.GPUStatus{ + Conditions: []metav1.Condition{ + {Type: "Healthy", Status: metav1.ConditionFalse, Reason: "XID79"}, + }, + }, + }, + want: "xid-error-check", + }, + { + name: "ECC error", + gpu: &devicev1alpha1.GPU{ + Status: devicev1alpha1.GPUStatus{ + Conditions: []metav1.Condition{ + {Type: "Healthy", Status: metav1.ConditionFalse, Reason: "ECCError"}, + }, + }, + }, + want: "memory-ecc-check", + }, + { + name: "thermal error", + gpu: &devicev1alpha1.GPU{ + Status: devicev1alpha1.GPUStatus{ + Conditions: []metav1.Condition{ + {Type: "Healthy", Status: metav1.ConditionFalse, Reason: "ThermalThrottle"}, + }, + }, + }, + want: "thermal-check", + }, + { + name: "unknown error", + gpu: &devicev1alpha1.GPU{ + Status: devicev1alpha1.GPUStatus{ + Conditions: []metav1.Condition{ + {Type: "Healthy", Status: metav1.ConditionFalse, Reason: "Unknown"}, + }, + }, + }, + want: "health-check", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := p.determineCheckName(tt.gpu); got != tt.want { + t.Errorf("determineCheckName() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPublisher_GenerateEventName(t *testing.T) { + p := &Publisher{ + config: Config{NodeName: "worker-1"}, + } + + gpu := &devicev1alpha1.GPU{ + Spec: devicev1alpha1.GPUSpec{ + UUID: "GPU-abc123def456", + }, + } + + name := p.generateEventName(gpu) + + // Should start with node name + if name[:8] != "worker-1" { + t.Errorf("generateEventName() should start with node name, got %s", name) + } + + // Should contain part of UUID + if len(name) < 20 { + t.Errorf("generateEventName() too short: %s", name) + } +} + +func TestPublisher_OnGPUUnhealthy_WithFakeClient(t *testing.T) { + scheme := runtime.NewScheme() + _ = nvsentinelv1alpha1.AddToScheme(scheme) + _ = devicev1alpha1.AddToScheme(scheme) + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + Build() + + p := &Publisher{ + config: Config{ + NodeName: "worker-1", + Enabled: true, + DebounceInterval: 100 * time.Millisecond, + Source: "test", + }, + client: fakeClient, + logger: klog.NewKlogr(), + lastPublish: make(map[string]time.Time), + pendingGPUs: make(map[string]*devicev1alpha1.GPU), + } + + gpu := &devicev1alpha1.GPU{ + Spec: devicev1alpha1.GPUSpec{ + UUID: "GPU-test-12345678", + }, + Status: devicev1alpha1.GPUStatus{ + Conditions: []metav1.Condition{ + { + Type: "Healthy", + Status: metav1.ConditionFalse, + Reason: "XID79", + Message: "Fatal GPU error", + }, + }, + RecommendedAction: "RESTART_BM", + }, + } + + ctx := context.Background() + p.OnGPUUnhealthy(ctx, gpu) + + // Wait for async publish + time.Sleep(200 * time.Millisecond) + + // Verify HealthEvent was created + var events nvsentinelv1alpha1.HealthEventList + if err := fakeClient.List(ctx, &events); err != nil { + t.Fatalf("Failed to list HealthEvents: %v", err) + } + + if len(events.Items) != 1 { + t.Errorf("Expected 1 HealthEvent, got %d", len(events.Items)) + return + } + + event := events.Items[0] + if event.Spec.NodeName != "worker-1" { + t.Errorf("HealthEvent.Spec.NodeName = %v, want worker-1", event.Spec.NodeName) + } + if !event.Spec.IsFatal { + t.Error("HealthEvent.Spec.IsFatal should be true") + } + if event.Status.Phase != nvsentinelv1alpha1.PhaseNew { + t.Errorf("HealthEvent.Status.Phase = %v, want New", event.Status.Phase) + } +} + +func TestPublisher_Debouncing(t *testing.T) { + scheme := runtime.NewScheme() + _ = nvsentinelv1alpha1.AddToScheme(scheme) + _ = devicev1alpha1.AddToScheme(scheme) + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + Build() + + p := &Publisher{ + config: Config{ + NodeName: "worker-1", + Enabled: true, + DebounceInterval: 500 * time.Millisecond, + Source: "test", + }, + client: fakeClient, + logger: klog.NewKlogr(), + lastPublish: make(map[string]time.Time), + pendingGPUs: make(map[string]*devicev1alpha1.GPU), + debounceStop: make(chan struct{}), + } + + gpu := &devicev1alpha1.GPU{ + Spec: devicev1alpha1.GPUSpec{ + UUID: "GPU-debounce-test", + }, + Status: devicev1alpha1.GPUStatus{ + Conditions: []metav1.Condition{ + {Type: "Healthy", Status: metav1.ConditionFalse, Reason: "XID79"}, + }, + RecommendedAction: "RESTART_BM", + }, + } + + ctx := context.Background() + + // First call should publish immediately + p.OnGPUUnhealthy(ctx, gpu) + time.Sleep(100 * time.Millisecond) + + // Second call should be debounced + p.OnGPUUnhealthy(ctx, gpu) + time.Sleep(100 * time.Millisecond) + + // Third call should still be debounced + p.OnGPUUnhealthy(ctx, gpu) + time.Sleep(100 * time.Millisecond) + + // Check only one event was created + var events nvsentinelv1alpha1.HealthEventList + if err := fakeClient.List(ctx, &events, &client.ListOptions{}); err != nil { + t.Fatalf("Failed to list HealthEvents: %v", err) + } + + if len(events.Items) != 1 { + t.Errorf("Expected 1 HealthEvent (debounced), got %d", len(events.Items)) + } +} diff --git a/pkg/deviceapiserver/metrics/metrics.go b/pkg/deviceapiserver/metrics/metrics.go new file mode 100644 index 000000000..eb0bd4c1c --- /dev/null +++ b/pkg/deviceapiserver/metrics/metrics.go @@ -0,0 +1,403 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package metrics provides Prometheus metrics for the Device API Server. +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" +) + +const ( + namespace = "device_api_server" +) + +// Cache operation labels. +const ( + OpRegister = "register" + OpUnregister = "unregister" + OpUpdateStatus = "update_status" + OpUpdateCondition = "update_condition" + OpSet = "set" +) + +// Watch event type labels. +const ( + EventAdded = "ADDED" + EventModified = "MODIFIED" + EventDeleted = "DELETED" +) + +// Metrics holds all Prometheus metrics for the Device API Server. +type Metrics struct { + // Server info + ServerInfo *prometheus.GaugeVec + + // Cache metrics + CacheGpusTotal prometheus.Gauge + CacheGpusHealthy prometheus.Gauge + CacheGpusUnhealthy prometheus.Gauge + CacheGpusUnknown prometheus.Gauge + CacheUpdatesTotal *prometheus.CounterVec + CacheResourceVersion prometheus.Gauge + + // Watch metrics + WatchStreamsActive prometheus.Gauge + WatchEventsTotal *prometheus.CounterVec + WatchEventsDropped prometheus.Counter + + // NVML provider metrics + NVMLProviderEnabled prometheus.Gauge + NVMLGpuCount prometheus.Gauge + NVMLHealthMonitorRunning prometheus.Gauge + + // Provider metrics (for containerized providers via heartbeat) + ProviderConnectionState *prometheus.GaugeVec + ProviderLastHeartbeat *prometheus.GaugeVec + ProviderHeartbeatTotal *prometheus.CounterVec + ProviderHeartbeatTimeouts *prometheus.CounterVec + ProviderGpusManaged *prometheus.GaugeVec + + // gRPC metrics (optional, for go-grpc-prometheus integration) + GRPCRequestsTotal *prometheus.CounterVec + GRPCRequestDuration *prometheus.HistogramVec + + // Registry for custom metrics + registry *prometheus.Registry +} + +// New creates and registers all metrics with a new registry. +func New() *Metrics { + m := &Metrics{ + registry: prometheus.NewRegistry(), + } + + // Server info + m.ServerInfo = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "info", + Help: "Server information with labels for version and node", + }, + []string{"version", "go_version", "node"}, + ) + + // Cache metrics + m.CacheGpusTotal = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: "cache", + Name: "gpus_total", + Help: "Total number of GPUs in cache", + }, + ) + + m.CacheGpusHealthy = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: "cache", + Name: "gpus_healthy", + Help: "Number of healthy GPUs", + }, + ) + + m.CacheGpusUnhealthy = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: "cache", + Name: "gpus_unhealthy", + Help: "Number of unhealthy GPUs", + }, + ) + + m.CacheGpusUnknown = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: "cache", + Name: "gpus_unknown", + Help: "Number of GPUs with unknown health state", + }, + ) + + m.CacheUpdatesTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "cache", + Name: "updates_total", + Help: "Total number of cache update operations", + }, + []string{"operation"}, + ) + + m.CacheResourceVersion = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: "cache", + Name: "resource_version", + Help: "Current cache resource version", + }, + ) + + // Watch metrics + m.WatchStreamsActive = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: "watch", + Name: "streams_active", + Help: "Number of active watch streams", + }, + ) + + m.WatchEventsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "watch", + Name: "events_total", + Help: "Total number of watch events sent", + }, + []string{"type"}, + ) + + m.WatchEventsDropped = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "watch", + Name: "events_dropped_total", + Help: "Total number of watch events dropped due to full subscriber buffers", + }, + ) + + // NVML provider metrics + m.NVMLProviderEnabled = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: "nvml", + Name: "provider_enabled", + Help: "Whether the NVML provider is enabled (1) or disabled (0)", + }, + ) + + m.NVMLGpuCount = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: "nvml", + Name: "gpu_count", + Help: "Number of GPUs discovered by NVML", + }, + ) + + m.NVMLHealthMonitorRunning = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: "nvml", + Name: "health_monitor_running", + Help: "Whether the NVML health monitor is running (1) or not (0)", + }, + ) + + // Provider metrics (for containerized providers via heartbeat) + m.ProviderConnectionState = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: "provider", + Name: "connection_state", + Help: "Provider connection state (1=connected, 0=disconnected)", + }, + []string{"provider_id"}, + ) + + m.ProviderLastHeartbeat = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: "provider", + Name: "last_heartbeat_timestamp_seconds", + Help: "Unix timestamp of last heartbeat from provider", + }, + []string{"provider_id"}, + ) + + m.ProviderHeartbeatTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "provider", + Name: "heartbeat_total", + Help: "Total number of heartbeats received from provider", + }, + []string{"provider_id"}, + ) + + m.ProviderHeartbeatTimeouts = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "provider", + Name: "heartbeat_timeout_total", + Help: "Total number of heartbeat timeouts per provider", + }, + []string{"provider_id"}, + ) + + m.ProviderGpusManaged = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: "provider", + Name: "gpus_managed", + Help: "Number of GPUs managed by provider", + }, + []string{"provider_id"}, + ) + + // gRPC metrics + m.GRPCRequestsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "grpc", + Name: "requests_total", + Help: "Total number of gRPC requests", + }, + []string{"method", "code"}, + ) + + m.GRPCRequestDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: "grpc", + Name: "request_duration_seconds", + Help: "Duration of gRPC requests in seconds", + Buckets: []float64{.001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10}, + }, + []string{"method"}, + ) + + // Register all metrics + m.registry.MustRegister( + // Standard Go collectors + collectors.NewGoCollector(), + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + + // Server info + m.ServerInfo, + + // Cache metrics + m.CacheGpusTotal, + m.CacheGpusHealthy, + m.CacheGpusUnhealthy, + m.CacheGpusUnknown, + m.CacheUpdatesTotal, + m.CacheResourceVersion, + + // Watch metrics + m.WatchStreamsActive, + m.WatchEventsTotal, + m.WatchEventsDropped, + + // NVML metrics + m.NVMLProviderEnabled, + m.NVMLGpuCount, + m.NVMLHealthMonitorRunning, + + // Provider metrics + m.ProviderConnectionState, + m.ProviderLastHeartbeat, + m.ProviderHeartbeatTotal, + m.ProviderHeartbeatTimeouts, + m.ProviderGpusManaged, + + // gRPC metrics + m.GRPCRequestsTotal, + m.GRPCRequestDuration, + ) + + return m +} + +// Registry returns the Prometheus registry for this metrics instance. +func (m *Metrics) Registry() *prometheus.Registry { + return m.registry +} + +// RecordCacheOperation increments the cache operations counter. +func (m *Metrics) RecordCacheOperation(operation string) { + m.CacheUpdatesTotal.WithLabelValues(operation).Inc() +} + +// RecordWatchEvent increments the watch events counter. +func (m *Metrics) RecordWatchEvent(eventType string) { + m.WatchEventsTotal.WithLabelValues(eventType).Inc() +} + +// RecordWatchEventDropped increments the dropped watch events counter. +func (m *Metrics) RecordWatchEventDropped() { + m.WatchEventsDropped.Inc() +} + +// UpdateCacheStats updates cache-related gauge metrics. +func (m *Metrics) UpdateCacheStats(total, healthy, unhealthy, unknown int, resourceVersion int64) { + m.CacheGpusTotal.Set(float64(total)) + m.CacheGpusHealthy.Set(float64(healthy)) + m.CacheGpusUnhealthy.Set(float64(unhealthy)) + m.CacheGpusUnknown.Set(float64(unknown)) + m.CacheResourceVersion.Set(float64(resourceVersion)) +} + +// UpdateWatchStreams updates the active watch streams gauge. +func (m *Metrics) UpdateWatchStreams(count int) { + m.WatchStreamsActive.Set(float64(count)) +} + +// UpdateNVMLStatus updates NVML provider metrics. +func (m *Metrics) UpdateNVMLStatus(enabled bool, gpuCount int, healthMonitorRunning bool) { + if enabled { + m.NVMLProviderEnabled.Set(1) + } else { + m.NVMLProviderEnabled.Set(0) + } + + m.NVMLGpuCount.Set(float64(gpuCount)) + + if healthMonitorRunning { + m.NVMLHealthMonitorRunning.Set(1) + } else { + m.NVMLHealthMonitorRunning.Set(0) + } +} + +// SetServerInfo sets the server info gauge. +func (m *Metrics) SetServerInfo(version, goVersion, node string) { + m.ServerInfo.WithLabelValues(version, goVersion, node).Set(1) +} + +// RecordProviderHeartbeat records a heartbeat from a provider. +func (m *Metrics) RecordProviderHeartbeat(providerID string, gpuCount int, timestampSeconds float64) { + m.ProviderConnectionState.WithLabelValues(providerID).Set(1) + m.ProviderLastHeartbeat.WithLabelValues(providerID).Set(timestampSeconds) + m.ProviderHeartbeatTotal.WithLabelValues(providerID).Inc() + m.ProviderGpusManaged.WithLabelValues(providerID).Set(float64(gpuCount)) +} + +// RecordProviderDisconnected marks a provider as disconnected. +func (m *Metrics) RecordProviderDisconnected(providerID string) { + m.ProviderConnectionState.WithLabelValues(providerID).Set(0) + m.ProviderHeartbeatTimeouts.WithLabelValues(providerID).Inc() +} + +// DeleteProviderMetrics removes metrics for a provider that has been cleaned up. +// This should be called after a provider has been disconnected for an extended period. +func (m *Metrics) DeleteProviderMetrics(providerID string) { + m.ProviderConnectionState.DeleteLabelValues(providerID) + m.ProviderLastHeartbeat.DeleteLabelValues(providerID) + m.ProviderGpusManaged.DeleteLabelValues(providerID) + // Note: Counter metrics (HeartbeatTotal, HeartbeatTimeouts) are not deleted + // to preserve historical data +} diff --git a/pkg/deviceapiserver/metrics/metrics_test.go b/pkg/deviceapiserver/metrics/metrics_test.go new file mode 100644 index 000000000..0a8bc7e77 --- /dev/null +++ b/pkg/deviceapiserver/metrics/metrics_test.go @@ -0,0 +1,172 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestNew(t *testing.T) { + m := New() + + if m == nil { + t.Fatal("New() returned nil") + } + + if m.Registry() == nil { + t.Error("Registry() returned nil") + } +} + +func TestSetServerInfo(t *testing.T) { + m := New() + m.SetServerInfo("v0.1.0", "go1.25.0", "node-1") + + // Verify metric was set + count := testutil.CollectAndCount(m.ServerInfo) + if count != 1 { + t.Errorf("Expected 1 server info metric, got %d", count) + } +} + +func TestUpdateCacheStats(t *testing.T) { + m := New() + m.UpdateCacheStats(10, 8, 1, 1, 100) + + // Verify total + total := testutil.ToFloat64(m.CacheGpusTotal) + if total != 10 { + t.Errorf("CacheGpusTotal = %f, want 10", total) + } + + // Verify healthy + healthy := testutil.ToFloat64(m.CacheGpusHealthy) + if healthy != 8 { + t.Errorf("CacheGpusHealthy = %f, want 8", healthy) + } + + // Verify unhealthy + unhealthy := testutil.ToFloat64(m.CacheGpusUnhealthy) + if unhealthy != 1 { + t.Errorf("CacheGpusUnhealthy = %f, want 1", unhealthy) + } + + // Verify unknown + unknown := testutil.ToFloat64(m.CacheGpusUnknown) + if unknown != 1 { + t.Errorf("CacheGpusUnknown = %f, want 1", unknown) + } + + // Verify resource version + rv := testutil.ToFloat64(m.CacheResourceVersion) + if rv != 100 { + t.Errorf("CacheResourceVersion = %f, want 100", rv) + } +} + +func TestRecordCacheOperation(t *testing.T) { + m := New() + + // Record some operations + m.RecordCacheOperation(OpRegister) + m.RecordCacheOperation(OpRegister) + m.RecordCacheOperation(OpUnregister) + m.RecordCacheOperation(OpUpdateStatus) + m.RecordCacheOperation(OpUpdateCondition) + + // Verify register count + expected := ` +# HELP device_api_server_cache_updates_total Total number of cache update operations +# TYPE device_api_server_cache_updates_total counter +device_api_server_cache_updates_total{operation="register"} 2 +device_api_server_cache_updates_total{operation="unregister"} 1 +device_api_server_cache_updates_total{operation="update_condition"} 1 +device_api_server_cache_updates_total{operation="update_status"} 1 +` + + err := testutil.CollectAndCompare(m.CacheUpdatesTotal, strings.NewReader(expected)) + if err != nil { + t.Errorf("CacheUpdatesTotal mismatch: %v", err) + } +} + +func TestUpdateWatchStreams(t *testing.T) { + m := New() + m.UpdateWatchStreams(5) + + value := testutil.ToFloat64(m.WatchStreamsActive) + if value != 5 { + t.Errorf("WatchStreamsActive = %f, want 5", value) + } +} + +func TestRecordWatchEvent(t *testing.T) { + m := New() + + m.RecordWatchEvent(EventAdded) + m.RecordWatchEvent(EventAdded) + m.RecordWatchEvent(EventModified) + m.RecordWatchEvent(EventDeleted) + + expected := ` +# HELP device_api_server_watch_events_total Total number of watch events sent +# TYPE device_api_server_watch_events_total counter +device_api_server_watch_events_total{type="ADDED"} 2 +device_api_server_watch_events_total{type="DELETED"} 1 +device_api_server_watch_events_total{type="MODIFIED"} 1 +` + + err := testutil.CollectAndCompare(m.WatchEventsTotal, strings.NewReader(expected)) + if err != nil { + t.Errorf("WatchEventsTotal mismatch: %v", err) + } +} + +func TestUpdateNVMLStatus(t *testing.T) { + m := New() + + // Test enabled + m.UpdateNVMLStatus(true, 4, true) + + if testutil.ToFloat64(m.NVMLProviderEnabled) != 1 { + t.Error("NVMLProviderEnabled should be 1 when enabled") + } + + if testutil.ToFloat64(m.NVMLGpuCount) != 4 { + t.Error("NVMLGpuCount should be 4") + } + + if testutil.ToFloat64(m.NVMLHealthMonitorRunning) != 1 { + t.Error("NVMLHealthMonitorRunning should be 1 when running") + } + + // Test disabled + m.UpdateNVMLStatus(false, 0, false) + + if testutil.ToFloat64(m.NVMLProviderEnabled) != 0 { + t.Error("NVMLProviderEnabled should be 0 when disabled") + } + + if testutil.ToFloat64(m.NVMLGpuCount) != 0 { + t.Error("NVMLGpuCount should be 0 when disabled") + } + + if testutil.ToFloat64(m.NVMLHealthMonitorRunning) != 0 { + t.Error("NVMLHealthMonitorRunning should be 0 when not running") + } +} diff --git a/pkg/deviceapiserver/nvml/enumerator.go b/pkg/deviceapiserver/nvml/enumerator.go new file mode 100644 index 000000000..178c16045 --- /dev/null +++ b/pkg/deviceapiserver/nvml/enumerator.go @@ -0,0 +1,203 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build nvml + +package nvml + +import ( + "fmt" + "time" + + "github.com/NVIDIA/go-nvml/pkg/nvml" + "google.golang.org/protobuf/types/known/timestamppb" + + v1alpha1 "github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1" +) + +// Condition status values (string-based per proto definition). +const ( + ConditionStatusTrue = "True" + ConditionStatusFalse = "False" + ConditionStatusUnknown = "Unknown" +) + +// enumerateDevices discovers all GPUs via NVML and registers them via gRPC. +// +// For each GPU found, it extracts device information and creates a GPU entry +// via the GpuService API with an initial "NVMLReady" condition set to True. +// +// Returns the number of GPUs discovered. +func (p *Provider) enumerateDevices() (int, error) { + count, ret := p.nvmllib.DeviceGetCount() + if ret != nvml.SUCCESS { + return 0, fmt.Errorf("failed to get device count: %v", nvml.ErrorString(ret)) + } + + if count == 0 { + p.logger.Info("No GPUs found on this node") + return 0, nil + } + + p.logger.V(1).Info("Enumerating GPUs", "count", count) + + successCount := 0 + p.gpuUUIDs = make([]string, 0, count) + + for i := 0; i < count; i++ { + device, ret := p.nvmllib.DeviceGetHandleByIndex(i) + if ret != nvml.SUCCESS { + p.logger.Error(nil, "Failed to get device handle", "index", i, "error", nvml.ErrorString(ret)) + + continue + } + + gpu, productName, memoryBytes, err := p.deviceToGpu(i, device) + if err != nil { + p.logger.Error(err, "Failed to get GPU info", "index", i) + + continue + } + + // Register GPU via gRPC (CreateGpu is idempotent) + resp, err := p.client.CreateGpu(p.ctx, &v1alpha1.CreateGpuRequest{Gpu: gpu}) + if err != nil { + p.logger.Error(err, "Failed to create GPU via gRPC", "uuid", gpu.GetMetadata().GetName()) + + continue + } + + // Track UUID for health monitoring + p.gpuUUIDs = append(p.gpuUUIDs, gpu.GetMetadata().GetName()) + + if resp.Created { + p.logger.Info("Created GPU", + "uuid", gpu.GetMetadata().GetName(), + "productName", productName, + "memory", formatBytes(memoryBytes), + ) + } else { + p.logger.V(1).Info("GPU already exists", + "uuid", gpu.GetMetadata().GetName(), + ) + } + + successCount++ + } + + return successCount, nil +} + +// deviceToGpu extracts GPU information from an NVML device handle. +// Returns the GPU proto, product name, and memory bytes (for logging). +func (p *Provider) deviceToGpu(index int, device Device) (*v1alpha1.Gpu, string, uint64, error) { + // Get UUID (required) + uuid, ret := device.GetUUID() + if ret != nvml.SUCCESS { + return nil, "", 0, fmt.Errorf("failed to get UUID: %v", nvml.ErrorString(ret)) + } + + // Get memory info (for logging) + var memoryBytes uint64 + + memInfo, ret := device.GetMemoryInfo() + if ret == nvml.SUCCESS { + memoryBytes = memInfo.Total + } + + // Get product name (for logging) + productName, ret := device.GetName() + if ret != nvml.SUCCESS { + productName = "Unknown" + } + + // Build GPU proto using available fields + // Note: Current proto only has uuid in GpuSpec, additional info stored in status message + now := timestamppb.Now() + gpu := &v1alpha1.Gpu{ + Metadata: &v1alpha1.ObjectMeta{Name: uuid}, + Spec: &v1alpha1.GpuSpec{ + Uuid: uuid, + }, + Status: &v1alpha1.GpuStatus{ + Conditions: []*v1alpha1.Condition{ + { + Type: ConditionTypeNVMLReady, + Status: ConditionStatusTrue, + Reason: "Initialized", + Message: fmt.Sprintf("GPU enumerated via NVML: %s (%s)", productName, formatBytes(memoryBytes)), + LastTransitionTime: now, + }, + }, + }, + } + + return gpu, productName, memoryBytes, nil +} + +// formatBytes formats bytes to human-readable string. +func formatBytes(bytes uint64) string { + const ( + KB = 1024 + MB = KB * 1024 + GB = MB * 1024 + ) + switch { + case bytes >= GB: + return fmt.Sprintf("%.1f GB", float64(bytes)/float64(GB)) + case bytes >= MB: + return fmt.Sprintf("%.1f MB", float64(bytes)/float64(MB)) + case bytes >= KB: + return fmt.Sprintf("%.1f KB", float64(bytes)/float64(KB)) + default: + return fmt.Sprintf("%d B", bytes) + } +} + +// Condition constants for NVML provider. +const ( + // ConditionTypeNVMLReady is the condition type for NVML health status. + ConditionTypeNVMLReady = "NVMLReady" + + // ConditionSourceNVML is the source identifier for conditions set by NVML provider. + ConditionSourceNVML = "nvml-provider" +) + +// UpdateCondition updates a condition on a GPU via the gRPC API. +// +// This completely replaces the GPU's status with a single condition. +// The condition's LastTransitionTime is set to the current time. +func (p *Provider) UpdateCondition( + uuid string, + conditionType string, + status string, + reason, message string, +) error { + condition := &v1alpha1.Condition{ + Type: conditionType, + Status: status, + Reason: reason, + Message: message, + LastTransitionTime: timestamppb.New(time.Now()), + } + + _, err := p.client.UpdateGpuStatus(p.ctx, &v1alpha1.UpdateGpuStatusRequest{ + Name: uuid, + Status: &v1alpha1.GpuStatus{ + Conditions: []*v1alpha1.Condition{condition}, + }, + }) + + return err +} diff --git a/pkg/deviceapiserver/nvml/health_monitor.go b/pkg/deviceapiserver/nvml/health_monitor.go new file mode 100644 index 000000000..7db66e64c --- /dev/null +++ b/pkg/deviceapiserver/nvml/health_monitor.go @@ -0,0 +1,264 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build nvml + +package nvml + +import ( + "fmt" + "time" + + "github.com/NVIDIA/go-nvml/pkg/nvml" +) + +// HealthMonitor monitors GPU health via NVML events. +type HealthMonitor struct { + provider *Provider +} + +// EventTimeout is the timeout for NVML event wait (in milliseconds). +const EventTimeout = 5000 + +// unknownUUID is used when UUID cannot be retrieved. +const unknownUUID = "unknown" + +// startHealthMonitoring initializes and starts XID event monitoring. +func (p *Provider) startHealthMonitoring() error { + // Create event set + eventSet, ret := p.nvmllib.EventSetCreate() + if ret != nvml.SUCCESS { + return fmt.Errorf("failed to create event set: %v", nvml.ErrorString(ret)) + } + + p.eventSet = eventSet + + // Register for health events on all GPUs + eventMask := uint64( + nvml.EventTypeXidCriticalError | + nvml.EventTypeDoubleBitEccError | + nvml.EventTypeSingleBitEccError, + ) + + count, _ := p.nvmllib.DeviceGetCount() + registeredCount := 0 + + for i := 0; i < count; i++ { + device, ret := p.nvmllib.DeviceGetHandleByIndex(i) + if ret != nvml.SUCCESS { + continue + } + + uuid, _ := device.GetUUID() + + // Get supported events for this device + supportedEvents, ret := device.GetSupportedEventTypes() + if ret != nvml.SUCCESS { + p.logger.V(1).Info("Device does not support event queries", + "index", i, + "uuid", uuid, + "error", nvml.ErrorString(ret), + ) + + continue + } + + // Register only supported events + eventsToRegister := eventMask & supportedEvents + if eventsToRegister == 0 { + p.logger.V(1).Info("Device does not support any health events", + "index", i, + "uuid", uuid, + ) + + continue + } + + ret = device.RegisterEvents(eventsToRegister, p.eventSet.Raw()) + if ret == nvml.ERROR_NOT_SUPPORTED { + p.logger.V(1).Info("Device too old for health monitoring", + "index", i, + "uuid", uuid, + ) + + continue + } + + if ret != nvml.SUCCESS { + p.logger.Error(nil, "Failed to register events", + "index", i, + "uuid", uuid, + "error", nvml.ErrorString(ret), + ) + + continue + } + + registeredCount++ + + p.logger.V(2).Info("Registered health events", + "index", i, + "uuid", uuid, + "events", eventsToRegister, + ) + } + + if registeredCount == 0 { + _ = p.eventSet.Free() + p.eventSet = nil + + return fmt.Errorf("no devices support health event monitoring") + } + + p.logger.Info("Starting health monitoring", "devices", registeredCount) + + // Create health monitor + p.healthMonitor = &HealthMonitor{provider: p} + + // Start monitoring goroutine + p.wg.Add(1) + + go p.runHealthMonitor() + + p.monitorRunning = true + + return nil +} + +// runHealthMonitor is the main health monitoring loop. +// +// The loop checks for context cancellation before each iteration to ensure +// prompt shutdown when requested. The processEvents() call blocks for up to +// EventTimeout milliseconds waiting for NVML events. +func (p *Provider) runHealthMonitor() { + defer p.wg.Done() + + p.logger.V(1).Info("Health monitor started") + + for { + // Check for shutdown before processing events. + // This ensures we respond promptly to cancellation rather than + // waiting for the next event timeout cycle. + select { + case <-p.ctx.Done(): + p.logger.V(1).Info("Health monitor stopping") + return + default: + } + + p.processEvents() + } +} + +// processEvents waits for and processes NVML events. +func (p *Provider) processEvents() { + event, ret := p.eventSet.Wait(EventTimeout) + + if ret == nvml.ERROR_TIMEOUT { + // Normal timeout, continue + return + } + + if ret != nvml.SUCCESS { + if ret == nvml.ERROR_GPU_IS_LOST { + p.logger.Error(nil, "GPU lost detected, marking all GPUs unhealthy") + p.markAllUnhealthy("GPULost", "GPU is lost error detected") + + return + } + + p.logger.V(2).Info("Error waiting for event", + "error", nvml.ErrorString(ret), + ) + + // Brief sleep to avoid tight loop on persistent errors + time.Sleep(100 * time.Millisecond) + + return + } + + // Process the event + p.handleEvent(event) +} + +// handleEvent processes a single NVML event. +func (p *Provider) handleEvent(event nvml.EventData) { + eventType := event.EventType + xid := event.EventData + gpuInstanceID := event.GpuInstanceId + computeInstanceID := event.ComputeInstanceId + + // Get UUID for logging + uuid := unknownUUID + + if event.Device != nil { + if u, ret := event.Device.GetUUID(); ret == nvml.SUCCESS { + uuid = u + } + } + + // Only process XID critical errors for health changes + if eventType != nvml.EventTypeXidCriticalError { + p.logger.V(2).Info("Non-critical event received", + "uuid", uuid, + "eventType", eventType, + "xid", xid, + ) + + return + } + + // Check if this XID should be ignored + if isIgnoredXid(xid, p.config.AdditionalIgnoredXids) { + p.logger.V(2).Info("Ignoring non-critical XID", + "uuid", uuid, + "xid", xid, + "gpuInstanceId", gpuInstanceID, + "computeInstanceId", computeInstanceID, + ) + + return + } + + // Critical XID - mark GPU unhealthy + p.logger.Info("Critical XID error detected", + "uuid", uuid, + "xid", xid, + "xidName", xidToString(xid), + "gpuInstanceId", gpuInstanceID, + "computeInstanceId", computeInstanceID, + ) + + message := fmt.Sprintf("Critical XID error %d (%s) detected", xid, xidToString(xid)) + if err := p.UpdateCondition(uuid, ConditionTypeNVMLReady, ConditionStatusFalse, "XidError", message); err != nil { + p.logger.Error(err, "Failed to update GPU condition", "uuid", uuid) + } +} + +// markAllUnhealthy marks all tracked GPUs as unhealthy. +func (p *Provider) markAllUnhealthy(reason, message string) { + for _, uuid := range p.gpuUUIDs { + err := p.UpdateCondition(uuid, ConditionTypeNVMLReady, ConditionStatusFalse, reason, message) + if err != nil { + p.logger.Error(err, "Failed to mark GPU unhealthy", "uuid", uuid) + } + } +} + +// MarkHealthy marks a specific GPU as healthy. +// +// This can be called to restore a GPU's health status after recovery. +func (p *Provider) MarkHealthy(uuid string) error { + return p.UpdateCondition(uuid, ConditionTypeNVMLReady, ConditionStatusTrue, "Healthy", "GPU is healthy") +} diff --git a/pkg/deviceapiserver/nvml/interface.go b/pkg/deviceapiserver/nvml/interface.go new file mode 100644 index 000000000..bc2ef995b --- /dev/null +++ b/pkg/deviceapiserver/nvml/interface.go @@ -0,0 +1,138 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build nvml + +package nvml + +import ( + "github.com/NVIDIA/go-nvml/pkg/nvml" +) + +// Library is the interface for NVML library operations. +// This interface contains only the methods used by the Provider, +// making it easier to mock for testing. +type Library interface { + Init() nvml.Return + Shutdown() nvml.Return + SystemGetDriverVersion() (string, nvml.Return) + DeviceGetCount() (int, nvml.Return) + DeviceGetHandleByIndex(index int) (Device, nvml.Return) + EventSetCreate() (EventSet, nvml.Return) +} + +// Device is the interface for NVML device operations. +type Device interface { + GetUUID() (string, nvml.Return) + GetName() (string, nvml.Return) + GetMemoryInfo() (nvml.Memory, nvml.Return) + GetSupportedEventTypes() (uint64, nvml.Return) + RegisterEvents(eventTypes uint64, set nvml.EventSet) nvml.Return +} + +// EventSet is the interface for NVML event set operations. +type EventSet interface { + Wait(timeout uint32) (nvml.EventData, nvml.Return) + Free() nvml.Return + // Raw returns the underlying nvml.EventSet for use with RegisterEvents. + Raw() nvml.EventSet +} + +// nvmlLibraryWrapper wraps the real nvml.Interface to implement Library. +type nvmlLibraryWrapper struct { + lib nvml.Interface +} + +// NewLibraryWrapper creates a Library wrapper around an nvml.Interface. +func NewLibraryWrapper(lib nvml.Interface) Library { + return &nvmlLibraryWrapper{lib: lib} +} + +func (w *nvmlLibraryWrapper) Init() nvml.Return { + return w.lib.Init() +} + +func (w *nvmlLibraryWrapper) Shutdown() nvml.Return { + return w.lib.Shutdown() +} + +func (w *nvmlLibraryWrapper) SystemGetDriverVersion() (string, nvml.Return) { + return w.lib.SystemGetDriverVersion() +} + +func (w *nvmlLibraryWrapper) DeviceGetCount() (int, nvml.Return) { + return w.lib.DeviceGetCount() +} + +func (w *nvmlLibraryWrapper) DeviceGetHandleByIndex(index int) (Device, nvml.Return) { + device, ret := w.lib.DeviceGetHandleByIndex(index) + if ret != nvml.SUCCESS { + return nil, ret + } + + return &nvmlDeviceWrapper{device: device}, ret +} + +func (w *nvmlLibraryWrapper) EventSetCreate() (EventSet, nvml.Return) { + es, ret := w.lib.EventSetCreate() + if ret != nvml.SUCCESS { + return nil, ret + } + + return &nvmlEventSetWrapper{es: es}, ret +} + +// nvmlDeviceWrapper wraps nvml.Device to implement Device. +type nvmlDeviceWrapper struct { + device nvml.Device +} + +func (w *nvmlDeviceWrapper) GetUUID() (string, nvml.Return) { + return w.device.GetUUID() +} + +func (w *nvmlDeviceWrapper) GetName() (string, nvml.Return) { + return w.device.GetName() +} + +func (w *nvmlDeviceWrapper) GetMemoryInfo() (nvml.Memory, nvml.Return) { + return w.device.GetMemoryInfo() +} + +func (w *nvmlDeviceWrapper) GetSupportedEventTypes() (uint64, nvml.Return) { + return w.device.GetSupportedEventTypes() +} + +func (w *nvmlDeviceWrapper) RegisterEvents(eventTypes uint64, set nvml.EventSet) nvml.Return { + return w.device.RegisterEvents(eventTypes, set) +} + +// nvmlEventSetWrapper wraps nvml.EventSet to implement EventSet. +type nvmlEventSetWrapper struct { + es nvml.EventSet +} + +func (w *nvmlEventSetWrapper) Wait(timeout uint32) (nvml.EventData, nvml.Return) { + return w.es.Wait(timeout) +} + +func (w *nvmlEventSetWrapper) Free() nvml.Return { + return w.es.Free() +} + +// Raw returns the underlying nvml.EventSet for use with device.RegisterEvents. +// This is needed because RegisterEvents expects the concrete nvml.EventSet type. +func (w *nvmlEventSetWrapper) Raw() nvml.EventSet { + return w.es +} diff --git a/pkg/deviceapiserver/nvml/mock_test.go b/pkg/deviceapiserver/nvml/mock_test.go new file mode 100644 index 000000000..6ccf6896d --- /dev/null +++ b/pkg/deviceapiserver/nvml/mock_test.go @@ -0,0 +1,237 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build nvml + +package nvml + +import ( + "sync" + + "github.com/NVIDIA/go-nvml/pkg/nvml" +) + +// MockLibrary is a mock implementation of Library for testing. +type MockLibrary struct { + // Init behavior + InitReturn nvml.Return + + // Shutdown behavior + ShutdownReturn nvml.Return + + // SystemGetDriverVersion behavior + DriverVersion string + DriverVersionReturn nvml.Return + + // DeviceGetCount behavior + DeviceCount int + DeviceCountReturn nvml.Return + + // Devices returns mock devices by index + Devices map[int]*MockDevice + + // EventSetCreate behavior + EventSet *MockEventSet + EventSetCreateReturn nvml.Return + + // Track calls for verification + mu sync.Mutex + InitCalled bool + ShutdownCalled bool +} + +// NewMockLibrary creates a new mock Library with defaults. +func NewMockLibrary() *MockLibrary { + return &MockLibrary{ + InitReturn: nvml.SUCCESS, + ShutdownReturn: nvml.SUCCESS, + DriverVersion: "535.104.05", + DriverVersionReturn: nvml.SUCCESS, + DeviceCount: 0, + DeviceCountReturn: nvml.SUCCESS, + Devices: make(map[int]*MockDevice), + EventSetCreateReturn: nvml.SUCCESS, + } +} + +// AddDevice adds a mock device at the specified index. +func (m *MockLibrary) AddDevice(index int, device *MockDevice) { + m.Devices[index] = device + m.DeviceCount = len(m.Devices) +} + +// Init implements Library. +func (m *MockLibrary) Init() nvml.Return { + m.mu.Lock() + defer m.mu.Unlock() + m.InitCalled = true + + return m.InitReturn +} + +// Shutdown implements Library. +func (m *MockLibrary) Shutdown() nvml.Return { + m.mu.Lock() + defer m.mu.Unlock() + m.ShutdownCalled = true + + return m.ShutdownReturn +} + +// SystemGetDriverVersion implements Library. +func (m *MockLibrary) SystemGetDriverVersion() (string, nvml.Return) { + return m.DriverVersion, m.DriverVersionReturn +} + +// DeviceGetCount implements Library. +func (m *MockLibrary) DeviceGetCount() (int, nvml.Return) { + return m.DeviceCount, m.DeviceCountReturn +} + +// DeviceGetHandleByIndex implements Library. +func (m *MockLibrary) DeviceGetHandleByIndex(index int) (Device, nvml.Return) { + if device, ok := m.Devices[index]; ok { + return device, nvml.SUCCESS + } + + return nil, nvml.ERROR_NOT_FOUND +} + +// EventSetCreate implements Library. +func (m *MockLibrary) EventSetCreate() (EventSet, nvml.Return) { + if m.EventSet == nil { + m.EventSet = NewMockEventSet() + } + + return m.EventSet, m.EventSetCreateReturn +} + +// MockDevice is a mock implementation of Device. +type MockDevice struct { + UUID string + UUIDReturn nvml.Return + Name string + NameReturn nvml.Return + MemoryInfo nvml.Memory + MemoryInfoReturn nvml.Return + SupportedEvents uint64 + SupportedEventsReturn nvml.Return + RegisterEventsReturn nvml.Return +} + +// NewMockDevice creates a new mock device with sensible defaults. +func NewMockDevice(uuid, name string) *MockDevice { + return &MockDevice{ + UUID: uuid, + UUIDReturn: nvml.SUCCESS, + Name: name, + NameReturn: nvml.SUCCESS, + MemoryInfo: nvml.Memory{ + Total: 16 * 1024 * 1024 * 1024, // 16 GB + Free: 15 * 1024 * 1024 * 1024, + Used: 1 * 1024 * 1024 * 1024, + }, + MemoryInfoReturn: nvml.SUCCESS, + SupportedEvents: uint64(nvml.EventTypeXidCriticalError | nvml.EventTypeDoubleBitEccError), + SupportedEventsReturn: nvml.SUCCESS, + RegisterEventsReturn: nvml.SUCCESS, + } +} + +// GetUUID implements Device. +func (d *MockDevice) GetUUID() (string, nvml.Return) { + return d.UUID, d.UUIDReturn +} + +// GetName implements Device. +func (d *MockDevice) GetName() (string, nvml.Return) { + return d.Name, d.NameReturn +} + +// GetMemoryInfo implements Device. +func (d *MockDevice) GetMemoryInfo() (nvml.Memory, nvml.Return) { + return d.MemoryInfo, d.MemoryInfoReturn +} + +// GetSupportedEventTypes implements Device. +func (d *MockDevice) GetSupportedEventTypes() (uint64, nvml.Return) { + return d.SupportedEvents, d.SupportedEventsReturn +} + +// RegisterEvents implements Device. +func (d *MockDevice) RegisterEvents(_ uint64, _ nvml.EventSet) nvml.Return { + return d.RegisterEventsReturn +} + +// MockEventSet is a mock implementation of EventSet. +type MockEventSet struct { + mu sync.Mutex + events []nvml.EventData + eventIdx int + WaitReturn nvml.Return + FreeReturn nvml.Return + Freed bool +} + +// NewMockEventSet creates a new mock event set. +func NewMockEventSet() *MockEventSet { + return &MockEventSet{ + events: make([]nvml.EventData, 0), + WaitReturn: nvml.ERROR_TIMEOUT, + FreeReturn: nvml.SUCCESS, + } +} + +// AddEvent adds an event to be returned by Wait. +func (e *MockEventSet) AddEvent(event nvml.EventData) { + e.mu.Lock() + defer e.mu.Unlock() + e.events = append(e.events, event) +} + +// Wait implements EventSet. +func (e *MockEventSet) Wait(_ uint32) (nvml.EventData, nvml.Return) { + e.mu.Lock() + defer e.mu.Unlock() + + if e.eventIdx < len(e.events) { + event := e.events[e.eventIdx] + e.eventIdx++ + + return event, nvml.SUCCESS + } + + return nvml.EventData{}, e.WaitReturn +} + +// Free implements EventSet. +func (e *MockEventSet) Free() nvml.Return { + e.mu.Lock() + defer e.mu.Unlock() + e.Freed = true + + return e.FreeReturn +} + +// Raw implements EventSet - returns nil for mocks since we don't need real event set. +func (e *MockEventSet) Raw() nvml.EventSet { + return nil +} + +// Compile-time interface checks. +var ( + _ Library = (*MockLibrary)(nil) + _ Device = (*MockDevice)(nil) + _ EventSet = (*MockEventSet)(nil) +) diff --git a/pkg/deviceapiserver/nvml/provider.go b/pkg/deviceapiserver/nvml/provider.go new file mode 100644 index 000000000..d4bfbb4a3 --- /dev/null +++ b/pkg/deviceapiserver/nvml/provider.go @@ -0,0 +1,290 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build nvml + +// Package nvml provides a built-in NVML-based health provider for the Device API Server. +// +// This provider uses NVML (NVIDIA Management Library) to: +// - Enumerate GPUs on the node at startup +// - Monitor GPU health via XID error events +// - Provide baseline device information when no external providers are connected +// +// The provider requires the NVIDIA driver to be installed and NVML libraries to be +// accessible. When running in Kubernetes, this is typically achieved by using the +// "nvidia" RuntimeClass which injects the driver libraries via the NVIDIA Container +// Toolkit, without consuming GPU resources. +package nvml + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + + "github.com/NVIDIA/go-nvml/pkg/nvml" + "k8s.io/klog/v2" + + v1alpha1 "github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1" +) + +// Provider is the built-in NVML-based health provider. +// +// It uses NVML to enumerate GPUs and monitor their health status. +// The provider is optional and gracefully degrades if NVML is unavailable. +// +// The provider communicates with the Device API Server via the gRPC client +// interface, making it a "dogfooding" client of its own API. This design: +// - Decouples the provider from server internals +// - Enables running the provider as a separate sidecar process +// - Validates the API from a provider's perspective +type Provider struct { + // Configuration + config Config + + // NVML library interface (uses our wrapper for testability) + nvmllib Library + + // gRPC client to communicate with Device API Server + client v1alpha1.GpuServiceClient + + // Logger + logger klog.Logger + + // Health monitoring + eventSet EventSet + healthMonitor *HealthMonitor + monitorRunning bool + + // Lifecycle management + mu sync.Mutex + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + + // State + initialized bool + gpuCount int + + // Tracked GPU UUIDs for health monitoring + gpuUUIDs []string +} + +// Config holds configuration for the NVML provider. +type Config struct { + // DriverRoot is the root path where NVIDIA driver libraries are located. + // Common values: + // - "/run/nvidia/driver" (container with CDI/RuntimeClass) + // - "/" (bare metal or host path mount) + DriverRoot string + + // AdditionalIgnoredXids is a list of additional XID error codes to ignore. + // These are added to the default list of ignored XIDs (application errors). + AdditionalIgnoredXids []uint64 + + // HealthCheckEnabled enables XID event monitoring for health checks. + // When disabled, only device enumeration is performed. + HealthCheckEnabled bool +} + +// DefaultConfig returns a Config with sensible defaults. +func DefaultConfig() Config { + return Config{ + DriverRoot: "/run/nvidia/driver", + AdditionalIgnoredXids: nil, + HealthCheckEnabled: true, + } +} + +// New creates a new NVML provider. +// +// The provider is not started until Start() is called. If NVML cannot be +// initialized (e.g., no driver installed), Start() will return an error +// but the server can continue without NVML support. +// +// The client parameter is a GpuServiceClient used to communicate with the +// Device API Server. This enables the provider to be either: +// - Co-located with the server (using a loopback connection) +// - Running as a separate sidecar process (using a network connection) +func New(cfg Config, client v1alpha1.GpuServiceClient, logger klog.Logger) *Provider { + logger = logger.WithName("nvml-provider") + + // Find NVML library path + libraryPath := findDriverLibrary(cfg.DriverRoot) + logger.V(2).Info("Using NVML library path", "path", libraryPath) + + // Create NVML interface with explicit library path + var rawLib nvml.Interface + if libraryPath != "" { + rawLib = nvml.New(nvml.WithLibraryPath(libraryPath)) + } else { + // Fall back to system default + rawLib = nvml.New() + } + + return &Provider{ + config: cfg, + nvmllib: NewLibraryWrapper(rawLib), + client: client, + logger: logger, + } +} + +// Start initializes NVML and enumerates GPUs. +// +// If health checking is enabled, it also starts the XID event monitoring +// goroutine. Returns an error if NVML cannot be initialized. +func (p *Provider) Start(ctx context.Context) error { + p.mu.Lock() + defer p.mu.Unlock() + + if p.initialized { + return fmt.Errorf("provider already started") + } + + p.logger.Info("Starting NVML provider") + + // Initialize NVML + ret := p.nvmllib.Init() + if ret != nvml.SUCCESS { + return fmt.Errorf("failed to initialize NVML: %v", nvml.ErrorString(ret)) + } + + // Get driver version for logging + driverVersion, ret := p.nvmllib.SystemGetDriverVersion() + if ret == nvml.SUCCESS { + p.logger.Info("NVML initialized", "driverVersion", driverVersion) + } + + // Set up context for lifecycle management before enumeration, + // since enumerateDevices uses p.ctx for gRPC calls. + p.ctx, p.cancel = context.WithCancel(ctx) + + // Enumerate devices + count, err := p.enumerateDevices() + if err != nil { + p.cancel() + _ = p.nvmllib.Shutdown() + + return fmt.Errorf("failed to enumerate devices: %w", err) + } + + p.gpuCount = count + + p.logger.Info("Enumerated GPUs", "count", count) + p.initialized = true + + // Start health monitoring if enabled and we have GPUs + if p.config.HealthCheckEnabled && count > 0 { + if err := p.startHealthMonitoring(); err != nil { + p.logger.Error(err, "Failed to start health monitoring, continuing without it") + // Don't fail - health monitoring is optional + } + } + + return nil +} + +// Stop shuts down the NVML provider. +// +// It stops health monitoring (if running) and shuts down NVML. +// This method is safe to call multiple times. +func (p *Provider) Stop() { + p.mu.Lock() + defer p.mu.Unlock() + + if !p.initialized { + return + } + + p.logger.Info("Stopping NVML provider") + + // Cancel context to stop health monitoring + if p.cancel != nil { + p.cancel() + } + + // Wait for health monitor to stop + p.wg.Wait() + + // Clean up event set + if p.eventSet != nil { + if ret := p.eventSet.Free(); ret != nvml.SUCCESS { + p.logger.V(1).Info("Failed to free event set", "error", nvml.ErrorString(ret)) + } + + p.eventSet = nil + } + + // Shutdown NVML + if ret := p.nvmllib.Shutdown(); ret != nvml.SUCCESS { + p.logger.V(1).Info("Failed to shutdown NVML", "error", nvml.ErrorString(ret)) + } + + p.initialized = false + p.monitorRunning = false + p.logger.Info("NVML provider stopped") +} + +// IsInitialized returns true if the provider has been successfully started. +func (p *Provider) IsInitialized() bool { + p.mu.Lock() + defer p.mu.Unlock() + + return p.initialized +} + +// GPUCount returns the number of GPUs discovered. +func (p *Provider) GPUCount() int { + p.mu.Lock() + defer p.mu.Unlock() + + return p.gpuCount +} + +// IsHealthMonitorRunning returns true if health monitoring is active. +func (p *Provider) IsHealthMonitorRunning() bool { + p.mu.Lock() + defer p.mu.Unlock() + + return p.monitorRunning +} + +// findDriverLibrary locates the NVML library in the driver root. +// +// It searches common paths where libnvidia-ml.so.1 might be located. +// Returns empty string if not found (will use system default). +func findDriverLibrary(driverRoot string) string { + if driverRoot == "" { + return "" + } + + // Common paths for libnvidia-ml.so.1 + searchPaths := []string{ + filepath.Join(driverRoot, "usr/lib64/libnvidia-ml.so.1"), + filepath.Join(driverRoot, "usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1"), + filepath.Join(driverRoot, "usr/lib/libnvidia-ml.so.1"), + filepath.Join(driverRoot, "lib64/libnvidia-ml.so.1"), + filepath.Join(driverRoot, "lib/libnvidia-ml.so.1"), + } + + for _, path := range searchPaths { + if _, err := os.Stat(path); err == nil { + return path + } + } + + return "" +} diff --git a/pkg/deviceapiserver/nvml/provider_test.go b/pkg/deviceapiserver/nvml/provider_test.go new file mode 100644 index 000000000..7a67fcae1 --- /dev/null +++ b/pkg/deviceapiserver/nvml/provider_test.go @@ -0,0 +1,511 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build nvml + +package nvml + +import ( + "context" + "testing" + "time" + + "github.com/NVIDIA/go-nvml/pkg/nvml" + "k8s.io/klog/v2" + + "github.com/nvidia/nvsentinel/pkg/deviceapiserver/cache" +) + +// testLogger returns a test logger. +func testLogger() klog.Logger { + return klog.NewKlogr().WithName("test") +} + +// TestProvider_Start_Success tests successful provider initialization. +func TestProvider_Start_Success(t *testing.T) { + mockLib := NewMockLibrary() + mockLib.AddDevice(0, NewMockDevice("GPU-uuid-0", "NVIDIA A100")) + mockLib.AddDevice(1, NewMockDevice("GPU-uuid-1", "NVIDIA A100")) + + gpuCache := cache.New(testLogger(), nil) + + provider := &Provider{ + config: DefaultConfig(), + nvmllib: mockLib, + cache: gpuCache, + logger: testLogger(), + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err := provider.Start(ctx) + if err != nil { + t.Fatalf("Start() failed: %v", err) + } + defer provider.Stop() + + // Verify NVML was initialized + if !mockLib.InitCalled { + t.Error("Init() was not called") + } + + // Verify GPUs were registered in cache + if gpuCache.Count() != 2 { + t.Errorf("Expected 2 GPUs in cache, got %d", gpuCache.Count()) + } + + // Verify provider state + if !provider.IsInitialized() { + t.Error("Provider should be initialized") + } + + if provider.GPUCount() != 2 { + t.Errorf("Expected GPUCount() = 2, got %d", provider.GPUCount()) + } +} + +// TestProvider_Start_NVMLInitFails tests graceful handling of NVML init failure. +func TestProvider_Start_NVMLInitFails(t *testing.T) { + mockLib := NewMockLibrary() + mockLib.InitReturn = nvml.ERROR_LIBRARY_NOT_FOUND + + gpuCache := cache.New(testLogger(), nil) + + provider := &Provider{ + config: DefaultConfig(), + nvmllib: mockLib, + cache: gpuCache, + logger: testLogger(), + } + + ctx := context.Background() + err := provider.Start(ctx) + + if err == nil { + t.Fatal("Expected Start() to fail when NVML init fails") + } + + if provider.IsInitialized() { + t.Error("Provider should not be initialized after failure") + } +} + +// TestProvider_Start_NoGPUs tests handling of nodes without GPUs. +func TestProvider_Start_NoGPUs(t *testing.T) { + mockLib := NewMockLibrary() + mockLib.DeviceCount = 0 + + gpuCache := cache.New(testLogger(), nil) + + provider := &Provider{ + config: DefaultConfig(), + nvmllib: mockLib, + cache: gpuCache, + logger: testLogger(), + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err := provider.Start(ctx) + if err != nil { + t.Fatalf("Start() failed: %v", err) + } + defer provider.Stop() + + if provider.GPUCount() != 0 { + t.Errorf("Expected 0 GPUs, got %d", provider.GPUCount()) + } + + // Health monitor should not be running with 0 GPUs + if provider.IsHealthMonitorRunning() { + t.Error("Health monitor should not run with 0 GPUs") + } +} + +// TestProvider_Start_AlreadyStarted tests double-start prevention. +func TestProvider_Start_AlreadyStarted(t *testing.T) { + mockLib := NewMockLibrary() + gpuCache := cache.New(testLogger(), nil) + + provider := &Provider{ + config: DefaultConfig(), + nvmllib: mockLib, + cache: gpuCache, + logger: testLogger(), + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // First start + err := provider.Start(ctx) + if err != nil { + t.Fatalf("First Start() failed: %v", err) + } + defer provider.Stop() + + // Second start should fail + err = provider.Start(ctx) + if err == nil { + t.Error("Second Start() should fail") + } +} + +// TestProvider_Stop tests provider shutdown. +func TestProvider_Stop(t *testing.T) { + mockLib := NewMockLibrary() + mockLib.AddDevice(0, NewMockDevice("GPU-uuid-0", "NVIDIA A100")) + + gpuCache := cache.New(testLogger(), nil) + + provider := &Provider{ + config: DefaultConfig(), + nvmllib: mockLib, + cache: gpuCache, + logger: testLogger(), + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err := provider.Start(ctx) + if err != nil { + t.Fatalf("Start() failed: %v", err) + } + + // Stop the provider + provider.Stop() + + // Verify state + if provider.IsInitialized() { + t.Error("Provider should not be initialized after Stop()") + } + + if !mockLib.ShutdownCalled { + t.Error("NVML Shutdown() was not called") + } + + // Double stop should be safe + provider.Stop() +} + +// TestProvider_Stop_NotStarted tests Stop() on unstarted provider. +func TestProvider_Stop_NotStarted(t *testing.T) { + mockLib := NewMockLibrary() + gpuCache := cache.New(testLogger(), nil) + + provider := &Provider{ + config: DefaultConfig(), + nvmllib: mockLib, + cache: gpuCache, + logger: testLogger(), + } + + // Stop should be safe even if not started + provider.Stop() + + if mockLib.ShutdownCalled { + t.Error("Shutdown() should not be called if provider was never started") + } +} + +// TestProvider_DeviceEnumeration tests that devices are properly enumerated. +func TestProvider_DeviceEnumeration(t *testing.T) { + mockLib := NewMockLibrary() + + // Add devices with varying configurations + device0 := NewMockDevice("GPU-11111111-1111-1111-1111-111111111111", "NVIDIA H100") + device0.MemoryInfo = nvml.Memory{Total: 80 * 1024 * 1024 * 1024} // 80 GB + + device1 := NewMockDevice("GPU-22222222-2222-2222-2222-222222222222", "NVIDIA A100") + device1.MemoryInfo = nvml.Memory{Total: 40 * 1024 * 1024 * 1024} // 40 GB + + mockLib.AddDevice(0, device0) + mockLib.AddDevice(1, device1) + + gpuCache := cache.New(testLogger(), nil) + + provider := &Provider{ + config: DefaultConfig(), + nvmllib: mockLib, + cache: gpuCache, + logger: testLogger(), + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err := provider.Start(ctx) + if err != nil { + t.Fatalf("Start() failed: %v", err) + } + defer provider.Stop() + + // Verify both devices are in cache + gpus := gpuCache.List() + if len(gpus) != 2 { + t.Fatalf("Expected 2 GPUs, got %d", len(gpus)) + } + + // Verify GPU details + uuids := make(map[string]bool) + for _, gpu := range gpus { + uuids[gpu.GetMetadata().GetName()] = true + + // Check initial condition + if gpu.Status == nil || len(gpu.Status.Conditions) == 0 { + t.Errorf("GPU %s has no conditions", gpu.GetMetadata().GetName()) + continue + } + + cond := gpu.Status.Conditions[0] + if cond.Type != ConditionTypeNVMLReady { + t.Errorf("Expected condition type %s, got %s", ConditionTypeNVMLReady, cond.Type) + } + + if cond.Status != ConditionStatusTrue { + t.Errorf("Expected condition status True, got %s", cond.Status) + } + } + + if !uuids["GPU-11111111-1111-1111-1111-111111111111"] { + t.Error("GPU-11111111... not found in cache") + } + + if !uuids["GPU-22222222-2222-2222-2222-222222222222"] { + t.Error("GPU-22222222... not found in cache") + } +} + +// TestProvider_DeviceEnumeration_PartialFailure tests handling of partial device failures. +func TestProvider_DeviceEnumeration_PartialFailure(t *testing.T) { + mockLib := NewMockLibrary() + + // First device is fine + mockLib.AddDevice(0, NewMockDevice("GPU-good", "NVIDIA A100")) + + // Second device fails UUID retrieval + device1 := NewMockDevice("GPU-bad", "NVIDIA A100") + device1.UUIDReturn = nvml.ERROR_UNKNOWN + mockLib.AddDevice(1, device1) + + // Third device is fine + mockLib.AddDevice(2, NewMockDevice("GPU-good-2", "NVIDIA A100")) + + gpuCache := cache.New(testLogger(), nil) + + provider := &Provider{ + config: DefaultConfig(), + nvmllib: mockLib, + cache: gpuCache, + logger: testLogger(), + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err := provider.Start(ctx) + if err != nil { + t.Fatalf("Start() failed: %v", err) + } + defer provider.Stop() + + // Only 2 GPUs should be in cache (one failed) + if gpuCache.Count() != 2 { + t.Errorf("Expected 2 GPUs (1 failed), got %d", gpuCache.Count()) + } +} + +// TestProvider_HealthCheckDisabled tests that health monitoring can be disabled. +func TestProvider_HealthCheckDisabled(t *testing.T) { + mockLib := NewMockLibrary() + mockLib.AddDevice(0, NewMockDevice("GPU-uuid-0", "NVIDIA A100")) + + gpuCache := cache.New(testLogger(), nil) + + config := DefaultConfig() + config.HealthCheckEnabled = false + + provider := &Provider{ + config: config, + nvmllib: mockLib, + cache: gpuCache, + logger: testLogger(), + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err := provider.Start(ctx) + if err != nil { + t.Fatalf("Start() failed: %v", err) + } + defer provider.Stop() + + // Give a moment for any goroutines to start + time.Sleep(10 * time.Millisecond) + + if provider.IsHealthMonitorRunning() { + t.Error("Health monitor should not be running when disabled") + } +} + +// TestProvider_UpdateCondition tests condition updates. +func TestProvider_UpdateCondition(t *testing.T) { + mockLib := NewMockLibrary() + mockLib.AddDevice(0, NewMockDevice("GPU-uuid-0", "NVIDIA A100")) + + gpuCache := cache.New(testLogger(), nil) + + config := DefaultConfig() + config.HealthCheckEnabled = false + + provider := &Provider{ + config: config, + nvmllib: mockLib, + cache: gpuCache, + logger: testLogger(), + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err := provider.Start(ctx) + if err != nil { + t.Fatalf("Start() failed: %v", err) + } + defer provider.Stop() + + // Update condition to unhealthy + err = provider.UpdateCondition("GPU-uuid-0", ConditionTypeNVMLReady, ConditionStatusFalse, "XidError", "Critical XID 48") + if err != nil { + t.Fatalf("UpdateCondition() failed: %v", err) + } + + // Verify condition was updated + gpu, found := gpuCache.Get("GPU-uuid-0") + if !found { + t.Fatal("GPU not found in cache") + } + + var foundCondition bool + + for _, cond := range gpu.Status.Conditions { + if cond.Type == ConditionTypeNVMLReady { + foundCondition = true + + if cond.Status != ConditionStatusFalse { + t.Errorf("Expected status False, got %s", cond.Status) + } + + if cond.Reason != "XidError" { + t.Errorf("Expected reason XidError, got %s", cond.Reason) + } + } + } + + if !foundCondition { + t.Error("NVMLReady condition not found") + } +} + +// TestProvider_UpdateCondition_GPUNotFound tests condition update for non-existent GPU. +func TestProvider_UpdateCondition_GPUNotFound(t *testing.T) { + mockLib := NewMockLibrary() + gpuCache := cache.New(testLogger(), nil) + + config := DefaultConfig() + config.HealthCheckEnabled = false + + provider := &Provider{ + config: config, + nvmllib: mockLib, + cache: gpuCache, + logger: testLogger(), + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err := provider.Start(ctx) + if err != nil { + t.Fatalf("Start() failed: %v", err) + } + defer provider.Stop() + + // Try to update condition for non-existent GPU + err = provider.UpdateCondition("GPU-nonexistent", ConditionTypeNVMLReady, ConditionStatusFalse, "XidError", "Test") + if err == nil { + t.Error("Expected error for non-existent GPU") + } +} + +// TestProvider_MarkHealthy tests marking a GPU as healthy. +func TestProvider_MarkHealthy(t *testing.T) { + mockLib := NewMockLibrary() + mockLib.AddDevice(0, NewMockDevice("GPU-uuid-0", "NVIDIA A100")) + + gpuCache := cache.New(testLogger(), nil) + + config := DefaultConfig() + config.HealthCheckEnabled = false + + provider := &Provider{ + config: config, + nvmllib: mockLib, + cache: gpuCache, + logger: testLogger(), + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err := provider.Start(ctx) + if err != nil { + t.Fatalf("Start() failed: %v", err) + } + defer provider.Stop() + + // First mark as unhealthy + err = provider.UpdateCondition("GPU-uuid-0", ConditionTypeNVMLReady, ConditionStatusFalse, "XidError", "Test") + if err != nil { + t.Fatalf("UpdateCondition() failed: %v", err) + } + + // Then mark as healthy + err = provider.MarkHealthy("GPU-uuid-0") + if err != nil { + t.Fatalf("MarkHealthy() failed: %v", err) + } + + // Verify it's healthy + gpu, found := gpuCache.Get("GPU-uuid-0") + if !found { + t.Fatal("GPU not found") + } + + for _, cond := range gpu.Status.Conditions { + if cond.Type == ConditionTypeNVMLReady { + if cond.Status != ConditionStatusTrue { + t.Errorf("Expected status True after MarkHealthy, got %s", cond.Status) + } + + return + } + } + + t.Error("NVMLReady condition not found") +} diff --git a/pkg/deviceapiserver/nvml/stub.go b/pkg/deviceapiserver/nvml/stub.go new file mode 100644 index 000000000..e6d4f165d --- /dev/null +++ b/pkg/deviceapiserver/nvml/stub.go @@ -0,0 +1,84 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !nvml + +// Package nvml provides a built-in NVML-based health provider for the Device API Server. +// +// This stub file is used when NVML support is not compiled in (build without -tags=nvml). +package nvml + +import ( + "context" + "errors" + + "k8s.io/klog/v2" + + v1alpha1 "github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1" +) + +// ErrNVMLNotCompiled is returned when NVML support is not compiled into the binary. +var ErrNVMLNotCompiled = errors.New("NVML support not compiled in (build with -tags=nvml)") + +// Provider is the built-in NVML-based health provider (stub when not compiled). +type Provider struct{} + +// Config holds configuration for the NVML provider. +type Config struct { + DriverRoot string + AdditionalIgnoredXids []uint64 + HealthCheckEnabled bool +} + +// DefaultConfig returns a Config with sensible defaults. +func DefaultConfig() Config { + return Config{ + DriverRoot: "/run/nvidia/driver", + AdditionalIgnoredXids: nil, + HealthCheckEnabled: true, + } +} + +// New creates a new NVML provider (stub). +func New(cfg Config, client v1alpha1.GpuServiceClient, logger klog.Logger) *Provider { + return &Provider{} +} + +// Start initializes NVML (stub - always returns error). +func (p *Provider) Start(ctx context.Context) error { + return ErrNVMLNotCompiled +} + +// Stop shuts down the NVML provider (stub - no-op). +func (p *Provider) Stop() {} + +// IsInitialized returns false (stub). +func (p *Provider) IsInitialized() bool { + return false +} + +// GPUCount returns 0 (stub). +func (p *Provider) GPUCount() int { + return 0 +} + +// IsHealthMonitorRunning returns false (stub). +func (p *Provider) IsHealthMonitorRunning() bool { + return false +} + +// ParseIgnoredXids parses a comma-separated string of XID values (stub). +func ParseIgnoredXids(input string) []uint64 { + return nil +} diff --git a/pkg/deviceapiserver/nvml/xid.go b/pkg/deviceapiserver/nvml/xid.go new file mode 100644 index 000000000..533487df0 --- /dev/null +++ b/pkg/deviceapiserver/nvml/xid.go @@ -0,0 +1,205 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build nvml + +package nvml + +// XID errors documentation: +// https://docs.nvidia.com/deploy/xid-errors/index.html + +// DefaultIgnoredXids contains XID error codes that are typically caused by +// application errors rather than hardware failures. These are ignored by +// default to avoid false positives in health monitoring. +// +// Reference: https://docs.nvidia.com/deploy/xid-errors/index.html#topic_4 +var DefaultIgnoredXids = map[uint64]bool{ + // Application errors - GPU should still be healthy + 13: true, // Graphics Engine Exception + 31: true, // GPU memory page fault + 43: true, // GPU stopped processing + 45: true, // Preemptive cleanup, due to previous errors + 68: true, // Video processor exception + 109: true, // Context Switch Timeout Error +} + +// CriticalXids contains XID error codes that indicate critical hardware +// failures requiring immediate attention. +var CriticalXids = map[uint64]bool{ + // Memory errors + 48: true, // Double Bit ECC Error + 63: true, // Row remapping failure + 64: true, // Uncontained ECC error + 74: true, // NVLink error + 79: true, // GPU has fallen off the bus + + // Fatal errors + 94: true, // Contained ECC error (severe) + 95: true, // Uncontained ECC error + 119: true, // GSP (GPU System Processor) error + 120: true, // GSP firmware error +} + +// XidDescriptions provides human-readable descriptions for common XIDs. +var XidDescriptions = map[uint64]string{ + // Application errors (typically ignored) + 13: "Graphics Engine Exception", + 31: "GPU memory page fault", + 43: "GPU stopped processing", + 45: "Preemptive cleanup", + 68: "Video processor exception", + 109: "Context Switch Timeout", + + // Memory errors + 48: "Double Bit ECC Error", + 63: "Row remapping failure", + 64: "Uncontained ECC error", + 74: "NVLink error", + 79: "GPU has fallen off the bus", + 94: "Contained ECC error", + 95: "Uncontained ECC error", + + // Other notable XIDs + 8: "GPU not accessible", + 32: "Invalid or corrupted push buffer stream", + 38: "Driver firmware error", + 56: "Display engine error", + 57: "Error programming video memory interface", + 62: "Internal micro-controller halt (non-fatal)", + 69: "Graphics engine accessor error", + 119: "GSP error", + 120: "GSP firmware error", +} + +// isIgnoredXid returns true if the XID should be ignored for health purposes. +// +// An XID is ignored if it's in the default ignored list OR in the additional +// ignored list provided by the user. +func isIgnoredXid(xid uint64, additionalIgnored []uint64) bool { + // Check default ignored list + if DefaultIgnoredXids[xid] { + return true + } + + // Check additional ignored list + for _, ignoredXid := range additionalIgnored { + if xid == ignoredXid { + return true + } + } + + return false +} + +// IsCriticalXid returns true if the XID indicates a critical hardware failure. +func IsCriticalXid(xid uint64) bool { + return CriticalXids[xid] +} + +// xidToString returns a human-readable description for an XID. +func xidToString(xid uint64) string { + if desc, ok := XidDescriptions[xid]; ok { + return desc + } + + return "Unknown XID" +} + +// ParseIgnoredXids parses a comma-separated string of XID values. +// +// Invalid values are skipped with a warning log. +func ParseIgnoredXids(input string) []uint64 { + if input == "" { + return nil + } + + var ( + result []uint64 + current uint64 + hasDigit bool + ) + + for _, ch := range input { + switch { + case ch >= '0' && ch <= '9': + current = current*10 + uint64(ch-'0') + hasDigit = true + case ch == ',' || ch == ' ': + if hasDigit { + result = append(result, current) + current = 0 + hasDigit = false + } + } + } + + // Don't forget the last value + if hasDigit { + result = append(result, current) + } + + return result +} + +// XidSeverity represents the severity level of an XID error. +type XidSeverity int + +const ( + // XidSeverityUnknown indicates the XID severity is unknown. + XidSeverityUnknown XidSeverity = iota + // XidSeverityIgnored indicates the XID is typically caused by applications. + XidSeverityIgnored + // XidSeverityWarning indicates the XID may indicate a problem. + XidSeverityWarning + // XidSeverityCritical indicates the XID indicates a critical hardware failure. + XidSeverityCritical +) + +// Severity string constants. +const ( + severityUnknown = "unknown" + severityIgnored = "ignored" + severityWarning = "warning" + severityCritical = "critical" +) + +// GetXidSeverity returns the severity level for an XID. +func GetXidSeverity(xid uint64) XidSeverity { + if DefaultIgnoredXids[xid] { + return XidSeverityIgnored + } + + if CriticalXids[xid] { + return XidSeverityCritical + } + + // XIDs not in either list are treated as warnings + return XidSeverityWarning +} + +// String returns a string representation of XidSeverity. +func (s XidSeverity) String() string { + switch s { + case XidSeverityUnknown: + return severityUnknown + case XidSeverityIgnored: + return severityIgnored + case XidSeverityWarning: + return severityWarning + case XidSeverityCritical: + return severityCritical + default: + return severityUnknown + } +} diff --git a/pkg/deviceapiserver/nvml/xid_test.go b/pkg/deviceapiserver/nvml/xid_test.go new file mode 100644 index 000000000..63360692c --- /dev/null +++ b/pkg/deviceapiserver/nvml/xid_test.go @@ -0,0 +1,251 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build nvml + +package nvml + +import ( + "testing" +) + +func TestIsIgnoredXid_DefaultIgnored(t *testing.T) { + // Test default ignored XIDs + defaultIgnored := []uint64{13, 31, 43, 45, 68, 109} + + for _, xid := range defaultIgnored { + if !isIgnoredXid(xid, nil) { + t.Errorf("XID %d should be ignored by default", xid) + } + } +} + +func TestIsIgnoredXid_CriticalNotIgnored(t *testing.T) { + // Test critical XIDs are not ignored by default + criticalXids := []uint64{48, 63, 64, 74, 79, 94, 95, 119, 120} + + for _, xid := range criticalXids { + if isIgnoredXid(xid, nil) { + t.Errorf("Critical XID %d should not be ignored by default", xid) + } + } +} + +func TestIsIgnoredXid_AdditionalIgnored(t *testing.T) { + // Test additional ignored XIDs + additionalIgnored := []uint64{48, 63} // Make critical XIDs ignored + + // Normally critical, but now ignored + if !isIgnoredXid(48, additionalIgnored) { + t.Error("XID 48 should be ignored when in additional list") + } + + if !isIgnoredXid(63, additionalIgnored) { + t.Error("XID 63 should be ignored when in additional list") + } + + // Still critical (not in additional list) + if isIgnoredXid(64, additionalIgnored) { + t.Error("XID 64 should not be ignored (not in additional list)") + } +} + +func TestIsIgnoredXid_UnknownXid(t *testing.T) { + // Unknown XIDs should not be ignored + unknownXids := []uint64{1, 2, 3, 999, 12345} + + for _, xid := range unknownXids { + if isIgnoredXid(xid, nil) { + t.Errorf("Unknown XID %d should not be ignored", xid) + } + } +} + +func TestIsCriticalXid(t *testing.T) { + tests := []struct { + xid uint64 + expected bool + }{ + // Critical XIDs + {48, true}, + {63, true}, + {64, true}, + {74, true}, + {79, true}, + {94, true}, + {95, true}, + {119, true}, + {120, true}, + + // Non-critical XIDs + {13, false}, + {31, false}, + {43, false}, + {1, false}, + {999, false}, + } + + for _, tt := range tests { + result := IsCriticalXid(tt.xid) + if result != tt.expected { + t.Errorf("IsCriticalXid(%d) = %v, want %v", tt.xid, result, tt.expected) + } + } +} + +func TestXidToString(t *testing.T) { + tests := []struct { + xid uint64 + expected string + }{ + {13, "Graphics Engine Exception"}, + {31, "GPU memory page fault"}, + {48, "Double Bit ECC Error"}, + {79, "GPU has fallen off the bus"}, + {109, "Context Switch Timeout"}, + {999, "Unknown XID"}, + {0, "Unknown XID"}, + } + + for _, tt := range tests { + result := xidToString(tt.xid) + if result != tt.expected { + t.Errorf("xidToString(%d) = %q, want %q", tt.xid, result, tt.expected) + } + } +} + +func TestParseIgnoredXids(t *testing.T) { + tests := []struct { + name string + input string + expected []uint64 + }{ + { + name: "empty string", + input: "", + expected: nil, + }, + { + name: "single value", + input: "48", + expected: []uint64{48}, + }, + { + name: "multiple comma separated", + input: "48,63,64", + expected: []uint64{48, 63, 64}, + }, + { + name: "with spaces", + input: "48, 63, 64", + expected: []uint64{48, 63, 64}, + }, + { + name: "space separated", + input: "48 63 64", + expected: []uint64{48, 63, 64}, + }, + { + name: "mixed separators", + input: "48, 63 64,65", + expected: []uint64{48, 63, 64, 65}, + }, + { + name: "trailing comma", + input: "48,63,", + expected: []uint64{48, 63}, + }, + { + name: "leading comma", + input: ",48,63", + expected: []uint64{48, 63}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := ParseIgnoredXids(tt.input) + + if len(result) != len(tt.expected) { + t.Errorf("ParseIgnoredXids(%q) len = %d, want %d", tt.input, len(result), len(tt.expected)) + return + } + + for i, v := range result { + if v != tt.expected[i] { + t.Errorf("ParseIgnoredXids(%q)[%d] = %d, want %d", tt.input, i, v, tt.expected[i]) + } + } + }) + } +} + +func TestGetXidSeverity(t *testing.T) { + tests := []struct { + xid uint64 + expected XidSeverity + }{ + // Ignored (application errors) + {13, XidSeverityIgnored}, + {31, XidSeverityIgnored}, + {43, XidSeverityIgnored}, + {45, XidSeverityIgnored}, + {68, XidSeverityIgnored}, + {109, XidSeverityIgnored}, + + // Critical (hardware failures) + {48, XidSeverityCritical}, + {63, XidSeverityCritical}, + {64, XidSeverityCritical}, + {74, XidSeverityCritical}, + {79, XidSeverityCritical}, + {94, XidSeverityCritical}, + {95, XidSeverityCritical}, + {119, XidSeverityCritical}, + {120, XidSeverityCritical}, + + // Warning (unknown XIDs) + {1, XidSeverityWarning}, + {2, XidSeverityWarning}, + {999, XidSeverityWarning}, + } + + for _, tt := range tests { + result := GetXidSeverity(tt.xid) + if result != tt.expected { + t.Errorf("GetXidSeverity(%d) = %v, want %v", tt.xid, result, tt.expected) + } + } +} + +func TestXidSeverity_String(t *testing.T) { + tests := []struct { + severity XidSeverity + expected string + }{ + {XidSeverityUnknown, "unknown"}, + {XidSeverityIgnored, "ignored"}, + {XidSeverityWarning, "warning"}, + {XidSeverityCritical, "critical"}, + {XidSeverity(99), "unknown"}, // Invalid severity + } + + for _, tt := range tests { + result := tt.severity.String() + if result != tt.expected { + t.Errorf("XidSeverity(%d).String() = %q, want %q", tt.severity, result, tt.expected) + } + } +} diff --git a/pkg/deviceapiserver/server.go b/pkg/deviceapiserver/server.go new file mode 100644 index 000000000..cd21e0a43 --- /dev/null +++ b/pkg/deviceapiserver/server.go @@ -0,0 +1,736 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package deviceapiserver implements the Device API Server. +// +// The Device API Server is a node-local gRPC cache server that acts as an +// intermediary between providers (health monitors) and consumers (device +// plugins, DRA drivers). +package deviceapiserver + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "os" + "path/filepath" + "runtime" + "sync" + "sync/atomic" + "time" + + "github.com/prometheus/client_golang/prometheus/promhttp" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/health" + "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/reflection" + "k8s.io/klog/v2" + + v1alpha1 "github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1" + "github.com/nvidia/nvsentinel/pkg/deviceapiserver/cache" + "github.com/nvidia/nvsentinel/pkg/deviceapiserver/metrics" + "github.com/nvidia/nvsentinel/pkg/deviceapiserver/nvml" + "github.com/nvidia/nvsentinel/pkg/deviceapiserver/service" + "github.com/nvidia/nvsentinel/pkg/version" +) + +// HTTP server timeout configuration. +// These values follow security best practices to prevent slowloris attacks +// and resource exhaustion from slow clients. +const ( + // httpReadHeaderTimeout is the time allowed to read request headers. + httpReadHeaderTimeout = 5 * time.Second + + // httpReadTimeout is the maximum duration for reading the entire request. + httpReadTimeout = 10 * time.Second + + // httpWriteTimeout is the maximum duration before timing out writes of the response. + httpWriteTimeout = 10 * time.Second + + // httpIdleTimeout is the maximum time to wait for the next request when keep-alives are enabled. + httpIdleTimeout = 120 * time.Second +) + +// Server is the Device API Server. +type Server struct { + config Config + logger klog.Logger + + // Components + cache *cache.GpuCache + metrics *metrics.Metrics + grpcServer *grpc.Server + healthServer *health.Server + gpuService *service.GpuService + nvmlProvider *nvml.Provider + + // Listeners + tcpListener net.Listener + providerListener net.Listener + unixListener net.Listener + + // HTTP servers + healthHTTPServer *http.Server + metricsHTTPServer *http.Server + + // Lifecycle + mu sync.Mutex + started bool + stopped bool + ready atomic.Bool + shuttingDown atomic.Bool +} + +// New creates a new Device API Server. +func New(config Config, logger klog.Logger) *Server { + logger = logger.WithName("server") + + // Create metrics first (needed by cache) + m := metrics.New() + + // Set server info metric + versionInfo := version.Get() + m.SetServerInfo(versionInfo.Version, runtime.Version(), config.NodeName) + + // Create cache with metrics integration + gpuCache := cache.New(logger, m) + + // Wire up dropped event metrics + gpuCache.Broadcaster().SetOnEventDrop(m.RecordWatchEventDropped) + + // Create gRPC health server + healthServer := health.NewServer() + + // Create gRPC server with reflection for debugging + grpcServer := grpc.NewServer() + reflection.Register(grpcServer) + + // Create unified GpuService + gpuService := service.NewGpuService(gpuCache) + + // Register services + v1alpha1.RegisterGpuServiceServer(grpcServer, gpuService) + grpc_health_v1.RegisterHealthServer(grpcServer, healthServer) + + // Note: NVML provider is created in Start() after gRPC server is listening, + // because it needs a loopback gRPC client connection. + + return &Server{ + config: config, + logger: logger, + cache: gpuCache, + metrics: m, + grpcServer: grpcServer, + healthServer: healthServer, + gpuService: gpuService, + } +} + +// Start starts the server and blocks until the context is cancelled. +func (s *Server) Start(ctx context.Context) error { + s.mu.Lock() + + if s.started { + s.mu.Unlock() + + return errors.New("server already started") + } + + s.started = true + s.mu.Unlock() + + s.logger.Info("Starting server") + + // Start listeners + if err := s.startListeners(); err != nil { + return fmt.Errorf("failed to start listeners: %w", err) + } + + // Start HTTP servers + if err := s.startHTTPServers(); err != nil { + s.stopListeners() + return fmt.Errorf("failed to start HTTP servers: %w", err) + } + + // Start gRPC server + errCh := make(chan error, 2) + s.startGRPCServers(errCh) + + // Create and start NVML provider (if enabled) + // This is done after gRPC server is listening so we can create a loopback client. + if s.config.NVMLEnabled { + if err := s.createAndStartNVMLProvider(ctx); err != nil { + // NVML failure is not fatal - log and continue + s.logger.Error(err, "Failed to start NVML provider, continuing without NVML support") + } else { + s.logger.Info("NVML provider started", + "gpuCount", s.nvmlProvider.GPUCount(), + "healthMonitorRunning", s.nvmlProvider.IsHealthMonitorRunning(), + ) + } + } + + // Mark as ready (for gRPC health and HTTP readiness) + s.setReady(true) + s.logger.Info("Server is ready", + "grpcAddress", s.config.GRPCAddress, + "providerAddress", s.config.ProviderAddress, + "unixSocket", s.config.UnixSocket, + "healthPort", s.config.HealthPort, + "metricsPort", s.config.MetricsPort, + ) + + // Start background metrics updater + metricsCtx, metricsCancel := context.WithCancel(ctx) + go s.runMetricsUpdater(metricsCtx) + + // Wait for shutdown signal or error + select { + case <-ctx.Done(): + s.logger.Info("Received shutdown signal") + case err := <-errCh: + s.logger.Error(err, "Server error") + metricsCancel() + + return err + } + + metricsCancel() + + // Graceful shutdown + return s.shutdown() +} + +// setReady updates the server readiness state. +func (s *Server) setReady(ready bool) { + s.ready.Store(ready) + + var status grpc_health_v1.HealthCheckResponse_ServingStatus + if ready { + status = grpc_health_v1.HealthCheckResponse_SERVING + } else { + status = grpc_health_v1.HealthCheckResponse_NOT_SERVING + } + + s.healthServer.SetServingStatus("", status) + s.healthServer.SetServingStatus("nvidia.device.v1alpha1.GpuService", status) +} + +// startListeners starts the TCP and Unix socket listeners. +func (s *Server) startListeners() error { + lc := &net.ListenConfig{} + + if err := s.startTCPListener(lc); err != nil { + return err + } + + if err := s.startProviderListener(lc); err != nil { + return err + } + + if err := s.startUnixListener(lc); err != nil { + return err + } + + return nil +} + +// startTCPListener starts the TCP listener if configured. +func (s *Server) startTCPListener(lc *net.ListenConfig) error { + if s.config.GRPCAddress == "" { + return nil + } + + var err error + + s.tcpListener, err = lc.Listen(context.Background(), "tcp", s.config.GRPCAddress) + if err != nil { + return fmt.Errorf("failed to listen on TCP %s: %w", s.config.GRPCAddress, err) + } + + s.logger.V(1).Info("TCP listener started", "address", s.tcpListener.Addr()) + + return nil +} + +// startProviderListener starts the provider TCP listener if configured. +// This provides a separate endpoint for provider (sidecar) connections. +func (s *Server) startProviderListener(lc *net.ListenConfig) error { + if s.config.ProviderAddress == "" { + return nil + } + + var err error + + s.providerListener, err = lc.Listen(context.Background(), "tcp", s.config.ProviderAddress) + if err != nil { + s.stopListeners() + + return fmt.Errorf("failed to listen on provider TCP %s: %w", s.config.ProviderAddress, err) + } + + s.logger.V(1).Info("Provider TCP listener started", "address", s.providerListener.Addr()) + + return nil +} + +// startUnixListener starts the Unix socket listener if configured. +func (s *Server) startUnixListener(lc *net.ListenConfig) error { + if s.config.UnixSocket == "" { + return nil + } + + // Create socket directory if needed + socketDir := filepath.Dir(s.config.UnixSocket) + if err := os.MkdirAll(socketDir, 0750); err != nil { + s.stopListeners() + + return fmt.Errorf("failed to create socket directory %s: %w", socketDir, err) + } + + // Remove stale socket file + if err := os.Remove(s.config.UnixSocket); err != nil && !os.IsNotExist(err) { + s.stopListeners() + + return fmt.Errorf("failed to remove stale socket %s: %w", s.config.UnixSocket, err) + } + + var err error + + s.unixListener, err = lc.Listen(context.Background(), "unix", s.config.UnixSocket) + if err != nil { + s.stopListeners() + + return fmt.Errorf("failed to listen on Unix socket %s: %w", s.config.UnixSocket, err) + } + + // Set socket permissions + if err := os.Chmod(s.config.UnixSocket, s.config.UnixSocketPermissions); err != nil { + s.stopListeners() + + return fmt.Errorf("failed to set socket permissions: %w", err) + } + + s.logger.V(1).Info("Unix socket listener started", + "path", s.config.UnixSocket, + "permissions", fmt.Sprintf("%04o", s.config.UnixSocketPermissions), + ) + + return nil +} + +// startHTTPServers starts the health and metrics HTTP servers. +func (s *Server) startHTTPServers() error { + // Health server + healthMux := http.NewServeMux() + healthMux.HandleFunc("/healthz", s.handleHealthz) + healthMux.HandleFunc("/readyz", s.handleReadyz) + healthMux.HandleFunc("/livez", s.handleHealthz) // Alias for liveness + + s.healthHTTPServer = &http.Server{ + Addr: fmt.Sprintf(":%d", s.config.HealthPort), + Handler: healthMux, + ReadHeaderTimeout: httpReadHeaderTimeout, + ReadTimeout: httpReadTimeout, + WriteTimeout: httpWriteTimeout, + IdleTimeout: httpIdleTimeout, + } + + go func() { + s.logger.V(1).Info("Health HTTP server started", "port", s.config.HealthPort) + + if err := s.healthHTTPServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + s.logger.Error(err, "Health HTTP server error") + } + }() + + // Metrics server using Prometheus handler + metricsMux := http.NewServeMux() + metricsMux.Handle("/metrics", promhttp.InstrumentMetricHandler( + s.metrics.Registry(), + promhttp.HandlerFor( + s.metrics.Registry(), + promhttp.HandlerOpts{ + EnableOpenMetrics: true, + }, + ), + )) + + s.metricsHTTPServer = &http.Server{ + Addr: fmt.Sprintf(":%d", s.config.MetricsPort), + Handler: metricsMux, + ReadHeaderTimeout: httpReadHeaderTimeout, + ReadTimeout: httpReadTimeout, + WriteTimeout: httpWriteTimeout, + IdleTimeout: httpIdleTimeout, + } + + go func() { + s.logger.V(1).Info("Metrics HTTP server started", "port", s.config.MetricsPort) + + if err := s.metricsHTTPServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + s.logger.Error(err, "Metrics HTTP server error") + } + }() + + return nil +} + +// startGRPCServers starts serving gRPC on all listeners. +func (s *Server) startGRPCServers(errCh chan<- error) { + // Serve on TCP + if s.tcpListener != nil { + go func() { + s.logger.V(1).Info("gRPC server started on TCP", "address", s.tcpListener.Addr()) + + if err := s.grpcServer.Serve(s.tcpListener); err != nil { + errCh <- fmt.Errorf("gRPC TCP server error: %w", err) + } + }() + } + + // Serve on Provider TCP (for sidecar provider connections) + if s.providerListener != nil { + go func() { + s.logger.V(1).Info("gRPC server started on provider TCP", "address", s.providerListener.Addr()) + + if err := s.grpcServer.Serve(s.providerListener); err != nil { + errCh <- fmt.Errorf("gRPC provider TCP server error: %w", err) + } + }() + } + + // Serve on Unix socket + if s.unixListener != nil { + go func() { + s.logger.V(1).Info("gRPC server started on Unix socket", "path", s.config.UnixSocket) + + if err := s.grpcServer.Serve(s.unixListener); err != nil { + errCh <- fmt.Errorf("gRPC Unix socket server error: %w", err) + } + }() + } +} + +// shutdown performs graceful shutdown following Kubernetes best practices. +// +// Shutdown sequence: +// 1. Mark as not ready (gRPC health + HTTP readyz) +// 2. Wait for shutdown delay (allow k8s to propagate endpoint removal) +// 3. Stop accepting new connections +// 4. Wait for existing requests to complete (up to timeout) +// 5. Force close remaining connections +// 6. Clean up resources +func (s *Server) shutdown() error { + if !s.markStopped() { + return nil + } + + s.logger.Info("Starting graceful shutdown", + "shutdownDelay", s.config.ShutdownDelay, + "shutdownTimeout", s.config.ShutdownTimeout, + ) + + // Step 1: Mark as not ready + s.setReady(false) + s.logger.V(1).Info("Marked as not ready") + + // Step 2: Wait for shutdown delay + s.waitShutdownDelay() + + // Step 3-5: Shutdown servers with timeout + errs := s.shutdownServers() + + // Step 6: Clean up resources + s.cleanupResources() + + if len(errs) > 0 { + s.logger.Error(errs[0], "Shutdown completed with errors", "errorCount", len(errs)) + + return errs[0] + } + + s.logger.Info("Server shutdown complete") + + return nil +} + +// markStopped marks the server as stopped, returns false if already stopped. +func (s *Server) markStopped() bool { + s.mu.Lock() + defer s.mu.Unlock() + + if s.stopped { + return false + } + + s.stopped = true + s.shuttingDown.Store(true) + + return true +} + +// waitShutdownDelay waits for the configured shutdown delay. +func (s *Server) waitShutdownDelay() { + if s.config.ShutdownDelay > 0 { + s.logger.V(1).Info("Waiting for shutdown delay", "delay", s.config.ShutdownDelay) + time.Sleep(s.config.ShutdownDelay) + } +} + +// shutdownServers shuts down HTTP and gRPC servers with timeout. +func (s *Server) shutdownServers() []error { + ctx, cancel := context.WithTimeout(context.Background(), s.config.ShutdownTimeout) + defer cancel() + + var wg sync.WaitGroup + + errCh := make(chan error, 4) + + // Shutdown HTTP servers + wg.Add(1) + + go func() { + defer wg.Done() + + if err := s.healthHTTPServer.Shutdown(ctx); err != nil { + errCh <- fmt.Errorf("health HTTP shutdown error: %w", err) + } + }() + + wg.Add(1) + + go func() { + defer wg.Done() + + if err := s.metricsHTTPServer.Shutdown(ctx); err != nil { + errCh <- fmt.Errorf("metrics HTTP shutdown error: %w", err) + } + }() + + // Graceful stop gRPC server + wg.Add(1) + + go func() { + defer wg.Done() + + s.shutdownGRPCServer(ctx) + }() + + // Wait for all shutdowns + wg.Wait() + close(errCh) + + // Collect any errors + var errs []error + + for err := range errCh { + errs = append(errs, err) + } + + return errs +} + +// shutdownGRPCServer gracefully stops the gRPC server. +func (s *Server) shutdownGRPCServer(ctx context.Context) { + done := make(chan struct{}) + + go func() { + s.grpcServer.GracefulStop() + close(done) + }() + + select { + case <-done: + s.logger.V(1).Info("gRPC server stopped gracefully") + case <-ctx.Done(): + s.logger.Info("gRPC graceful stop timeout, forcing stop") + s.grpcServer.Stop() + } +} + +// createAndStartNVMLProvider creates the NVML provider with a loopback gRPC client +// and starts it. +// +// This is called after the gRPC server is listening so we can create a client +// connection to ourselves. The NVML provider uses this client to register GPUs +// and update health status via the standard GpuService API ("dogfooding"). +func (s *Server) createAndStartNVMLProvider(ctx context.Context) error { + // Determine the best endpoint for loopback connection + // Prefer Unix socket for lower latency, fall back to TCP + var target string + if s.config.UnixSocket != "" { + target = "unix://" + s.config.UnixSocket + } else { + target = s.config.GRPCAddress + } + + s.logger.V(1).Info("Creating NVML provider with loopback gRPC client", "target", target) + + // Create loopback gRPC client connection + conn, err := grpc.NewClient(target, + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + return fmt.Errorf("failed to create loopback gRPC connection: %w", err) + } + + // Create GpuService client + client := v1alpha1.NewGpuServiceClient(conn) + + // Create NVML provider configuration + nvmlConfig := nvml.Config{ + DriverRoot: s.config.NVMLDriverRoot, + AdditionalIgnoredXids: nvml.ParseIgnoredXids(s.config.NVMLIgnoredXids), + HealthCheckEnabled: s.config.NVMLHealthCheckEnabled, + } + + // Create and start NVML provider + s.nvmlProvider = nvml.New(nvmlConfig, client, s.logger) + if err := s.nvmlProvider.Start(ctx); err != nil { + conn.Close() + s.nvmlProvider = nil + return fmt.Errorf("failed to start NVML provider: %w", err) + } + + return nil +} + +// cleanupResources cleans up server resources. +func (s *Server) cleanupResources() { + // Stop NVML provider + if s.nvmlProvider != nil { + s.nvmlProvider.Stop() + s.logger.V(1).Info("NVML provider stopped") + } + + s.cache.Broadcaster().Close() + + // Clean up Unix socket + if s.config.UnixSocket != "" { + if err := os.Remove(s.config.UnixSocket); err != nil && !os.IsNotExist(err) { + s.logger.V(1).Info("Failed to remove Unix socket", + "path", s.config.UnixSocket, + "error", err, + ) + } + } +} + +// stopListeners closes all listeners. +func (s *Server) stopListeners() { + if s.tcpListener != nil { + s.tcpListener.Close() + } + + if s.providerListener != nil { + s.providerListener.Close() + } + + if s.unixListener != nil { + s.unixListener.Close() + } +} + +// handleHealthz handles liveness probe requests. +// Returns 200 OK if the process is alive (always true if we can respond). +func (s *Server) handleHealthz(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok\n")) +} + +// handleReadyz handles readiness probe requests. +// Returns 200 OK if ready to serve traffic, 503 during shutdown. +func (s *Server) handleReadyz(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + + // Check if shutting down + if s.shuttingDown.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte("shutting down\n")) + + return + } + + // Check gRPC health + if !s.ready.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte("not ready\n")) + + return + } + + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok\n")) +} + +// runMetricsUpdater periodically updates the metrics gauges. +func (s *Server) runMetricsUpdater(ctx context.Context) { + // Update interval for gauge metrics + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + // Initial update + s.updateMetrics() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.updateMetrics() + } + } +} + +// updateMetrics updates all gauge metrics from current state. +func (s *Server) updateMetrics() { + // Update cache stats + stats := s.cache.GetStats() + s.metrics.UpdateCacheStats( + stats.TotalGpus, + stats.HealthyGpus, + stats.UnhealthyGpus, + stats.UnknownGpus, + stats.ResourceVersion, + ) + + // Update watch streams + s.metrics.UpdateWatchStreams(s.cache.Broadcaster().SubscriberCount()) + + // Update NVML status + if s.nvmlProvider != nil && s.nvmlProvider.IsInitialized() { + s.metrics.UpdateNVMLStatus( + true, + s.nvmlProvider.GPUCount(), + s.nvmlProvider.IsHealthMonitorRunning(), + ) + } else { + s.metrics.UpdateNVMLStatus(false, 0, false) + } +} + +// Cache returns the GPU cache (for testing). +func (s *Server) Cache() *cache.GpuCache { + return s.cache +} + +// IsReady returns whether the server is ready to serve traffic. +func (s *Server) IsReady() bool { + return s.ready.Load() +} diff --git a/pkg/deviceapiserver/service/gpu_service.go b/pkg/deviceapiserver/service/gpu_service.go new file mode 100644 index 000000000..42ded7380 --- /dev/null +++ b/pkg/deviceapiserver/service/gpu_service.go @@ -0,0 +1,398 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package service implements the gRPC services for the Device API Server. +package service + +import ( + "context" + "errors" + "strconv" + + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + "k8s.io/klog/v2" + + v1alpha1 "github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1" + "github.com/nvidia/nvsentinel/pkg/deviceapiserver/cache" +) + +// GpuService implements the unified GpuService API. +// +// This service provides both read and write access to GPU resources: +// - Read operations (Get, List, Watch) for consumers like device plugins and DRA drivers +// - Write operations (Create, Update, UpdateStatus, Delete) for providers like health monitors +// +// All read operations acquire a read lock on the cache, which will block if a +// provider is currently updating GPU state. +// +// All write operations acquire an exclusive write lock, blocking ALL consumer +// read operations until the write completes. This ensures consumers never read +// stale data during status transitions. +type GpuService struct { + v1alpha1.UnimplementedGpuServiceServer + cache *cache.GpuCache +} + +// NewGpuService creates a new GpuService instance. +func NewGpuService(gpuCache *cache.GpuCache) *GpuService { + return &GpuService{ + cache: gpuCache, + } +} + +// ========================================== +// Read Operations +// ========================================== + +// GetGpu retrieves a single GPU resource by its unique name. +// +// This method acquires a read lock on the cache, blocking if a write is in progress. +// Returns NotFound if the GPU does not exist. +func (s *GpuService) GetGpu(ctx context.Context, req *v1alpha1.GetGpuRequest) (*v1alpha1.GetGpuResponse, error) { + logger := klog.FromContext(ctx).WithValues("method", "GetGpu", "gpuName", req.GetName()) + + // Validate request + if req.GetName() == "" { + logger.V(1).Info("Invalid request: empty name") + + return nil, status.Error(codes.InvalidArgument, "name is required") + } + + // Get GPU from cache (acquires read lock, blocks during writes) + gpu, found := s.cache.Get(req.GetName()) + if !found { + logger.V(1).Info("GPU not found") + + return nil, status.Error(codes.NotFound, "gpu not found") + } + + logger.V(2).Info("GPU retrieved successfully", "resourceVersion", gpu.GetMetadata().GetResourceVersion()) + + return &v1alpha1.GetGpuResponse{Gpu: gpu}, nil +} + +// ListGpus retrieves a list of all GPU resources. +// +// This method acquires a read lock on the cache, blocking if a write is in progress. +// Returns an empty list if no GPUs are registered. +// +// The response includes ListMeta.ResourceVersion which can be used for +// optimistic concurrency or cache invalidation. +func (s *GpuService) ListGpus(ctx context.Context, req *v1alpha1.ListGpusRequest) (*v1alpha1.ListGpusResponse, error) { + logger := klog.FromContext(ctx).WithValues("method", "ListGpus") + + // List all GPUs and resource version atomically under a single read lock + gpus, resourceVersion := s.cache.ListWithVersion() + + logger.V(2).Info("GPUs listed successfully", + "count", len(gpus), + "resourceVersion", resourceVersion, + ) + + return &v1alpha1.ListGpusResponse{ + GpuList: &v1alpha1.GpuList{ + Metadata: &v1alpha1.ListMeta{ + ResourceVersion: strconv.FormatInt(resourceVersion, 10), + }, + Items: gpus, + }, + }, nil +} + +// WatchGpus streams lifecycle events for GPU resources. +// +// The stream first sends ADDED events for all existing GPUs (initial sync), +// then streams MODIFIED and DELETED events as they occur. +// +// The stream remains open until: +// - The client disconnects +// - The server shuts down +// - An error occurs +func (s *GpuService) WatchGpus(req *v1alpha1.WatchGpusRequest, stream grpc.ServerStreamingServer[v1alpha1.WatchGpusResponse]) error { + ctx := stream.Context() + logger := klog.FromContext(ctx).WithValues("method", "WatchGpus") + + // Generate unique subscriber ID + subscriberID := uuid.New().String() + logger = logger.WithValues("subscriberID", subscriberID) + logger.V(1).Info("Watch stream started") + + // Atomically list existing GPUs and subscribe to events under a single lock. + // This prevents events from being missed or duplicated between the list + // snapshot and the start of the subscription. + gpus, events := s.cache.ListAndSubscribe(subscriberID) + + defer func() { + s.cache.Broadcaster().Unsubscribe(subscriberID) + logger.V(1).Info("Watch stream ended") + }() + + for _, gpu := range gpus { + if err := stream.Send(&v1alpha1.WatchGpusResponse{ + Type: cache.EventTypeAdded, + Object: gpu, + }); err != nil { + logger.Error(err, "Failed to send initial GPU state") + + return status.Error(codes.Internal, "failed to send initial state") + } + } + + logger.V(2).Info("Sent initial state", "gpuCount", len(gpus)) + + // Stream ongoing events + for { + select { + case <-ctx.Done(): + // Client disconnected or context cancelled + logger.V(1).Info("Watch stream context done", "reason", ctx.Err()) + return nil + + case event, ok := <-events: + if !ok { + // Channel closed (server shutting down) + logger.V(1).Info("Event channel closed") + return nil + } + + // Send event to client + if err := stream.Send(&v1alpha1.WatchGpusResponse{ + Type: event.Type, + Object: event.Object, + }); err != nil { + logger.Error(err, "Failed to send event", + "eventType", event.Type, + "gpuName", event.Object.GetMetadata().GetName(), + ) + + return status.Error(codes.Internal, "failed to send event") + } + + logger.V(2).Info("Event sent", + "eventType", event.Type, + "gpuName", event.Object.GetMetadata().GetName(), + "resourceVersion", event.Object.GetMetadata().GetResourceVersion(), + ) + } + } +} + +// ========================================== +// Write Operations +// ========================================== + +// CreateGpu registers a new GPU with the server. +// +// The GPU name (metadata.name) must be unique. If a GPU with the +// same name already exists, returns ALREADY_EXISTS. +// +// This operation acquires a write lock, blocking consumer reads +// until the operation completes. +func (s *GpuService) CreateGpu(ctx context.Context, req *v1alpha1.CreateGpuRequest) (*v1alpha1.CreateGpuResponse, error) { + logger := klog.FromContext(ctx).WithValues( + "method", "CreateGpu", + "gpuName", req.GetGpu().GetMetadata().GetName(), + ) + + // Validate request + if req.GetGpu() == nil { + logger.V(1).Info("Invalid request: nil gpu") + return nil, status.Error(codes.InvalidArgument, "gpu is required") + } + + if req.GetGpu().GetMetadata() == nil || req.GetGpu().GetMetadata().GetName() == "" { + logger.V(1).Info("Invalid request: empty gpu name") + return nil, status.Error(codes.InvalidArgument, "gpu.metadata.name is required") + } + + // Create GPU (acquires write lock, blocks consumers) + gpu, err := s.cache.Create(req.GetGpu()) + if err != nil { + if errors.Is(err, cache.ErrGpuAlreadyExists) { + logger.V(1).Info("GPU already exists") + // Return the existing GPU for idempotent registration + existing, _ := s.cache.Get(req.GetGpu().GetMetadata().GetName()) + return &v1alpha1.CreateGpuResponse{ + Gpu: existing, + Created: false, + }, nil + } + + logger.Error(err, "Failed to create GPU") + return nil, status.Errorf(codes.Internal, "failed to create gpu: %v", err) + } + + logger.Info("GPU created", "resourceVersion", gpu.GetMetadata().GetResourceVersion()) + + return &v1alpha1.CreateGpuResponse{ + Gpu: gpu, + Created: true, + }, nil +} + +// UpdateGpu replaces an existing GPU resource. +// +// The entire GPU (spec and status) is replaced. Use UpdateGpuStatus +// if you only need to update the status. +// +// Optimistic concurrency: If resource_version is provided in the GPU and does +// not match the current version, returns ABORTED. +// +// This operation acquires a write lock. +func (s *GpuService) UpdateGpu(ctx context.Context, req *v1alpha1.UpdateGpuRequest) (*v1alpha1.UpdateGpuResponse, error) { + logger := klog.FromContext(ctx).WithValues( + "method", "UpdateGpu", + "gpuName", req.GetGpu().GetMetadata().GetName(), + ) + + // Validate request + if req.GetGpu() == nil { + logger.V(1).Info("Invalid request: nil gpu") + return nil, status.Error(codes.InvalidArgument, "gpu is required") + } + + if req.GetGpu().GetMetadata() == nil || req.GetGpu().GetMetadata().GetName() == "" { + logger.V(1).Info("Invalid request: empty gpu name") + return nil, status.Error(codes.InvalidArgument, "gpu.metadata.name is required") + } + + // Update GPU (acquires write lock, blocks consumers) + var expectedVersion int64 + if rv := req.GetGpu().GetMetadata().GetResourceVersion(); rv != "" { + var err error + expectedVersion, err = strconv.ParseInt(rv, 10, 64) + if err != nil { + logger.V(1).Info("Invalid resource_version", "resourceVersion", rv, "error", err) + return nil, status.Errorf(codes.InvalidArgument, "invalid resource_version %q: must be a numeric string", rv) + } + } + gpu, err := s.cache.Update(req.GetGpu(), expectedVersion) + if err != nil { + if errors.Is(err, cache.ErrGpuNotFound) { + logger.V(1).Info("GPU not found") + return nil, status.Error(codes.NotFound, "gpu not found") + } + if errors.Is(err, cache.ErrConflict) { + logger.V(1).Info("Resource version conflict", "expectedVersion", expectedVersion) + return nil, status.Error(codes.Aborted, "resource version conflict") + } + + logger.Error(err, "Failed to update GPU") + return nil, status.Errorf(codes.Internal, "failed to update gpu: %v", err) + } + + logger.V(1).Info("GPU updated", "resourceVersion", gpu.GetMetadata().GetResourceVersion()) + + return &v1alpha1.UpdateGpuResponse{Gpu: gpu}, nil +} + +// UpdateGpuStatus updates only the status of an existing GPU. +// +// This is the primary method for health monitors to report GPU state. +// The spec is not modified. +// +// IMPORTANT: This operation acquires an exclusive write lock, blocking +// ALL consumer read operations (GetGpu, ListGpus) until the update completes. +// This ensures consumers never read stale data during status transitions. +// +// Optimistic concurrency: If resource_version is provided and does not match +// the current version, returns ABORTED. +func (s *GpuService) UpdateGpuStatus(ctx context.Context, req *v1alpha1.UpdateGpuStatusRequest) (*v1alpha1.UpdateGpuStatusResponse, error) { + logger := klog.FromContext(ctx).WithValues( + "method", "UpdateGpuStatus", + "gpuName", req.GetName(), + ) + + // Validate request + if req.GetName() == "" { + logger.V(1).Info("Invalid request: empty name") + return nil, status.Error(codes.InvalidArgument, "name is required") + } + + if req.GetStatus() == nil { + logger.V(1).Info("Invalid request: nil status") + return nil, status.Error(codes.InvalidArgument, "status is required") + } + + // Update status (acquires write lock, BLOCKS ALL CONSUMER READS) + var expectedVersion int64 + if rv := req.GetResourceVersion(); rv != "" { + var err error + expectedVersion, err = strconv.ParseInt(rv, 10, 64) + if err != nil { + logger.V(1).Info("Invalid resource_version", "resourceVersion", rv, "error", err) + return nil, status.Errorf(codes.InvalidArgument, "invalid resource_version %q: must be a numeric string", rv) + } + } + gpu, err := s.cache.UpdateStatusWithVersion(req.GetName(), req.GetStatus(), expectedVersion) + if err != nil { + if errors.Is(err, cache.ErrGpuNotFound) { + logger.V(1).Info("GPU not found") + return nil, status.Error(codes.NotFound, "gpu not found") + } + if errors.Is(err, cache.ErrConflict) { + logger.V(1).Info("Resource version conflict", "expectedVersion", req.GetResourceVersion()) + return nil, status.Error(codes.Aborted, "resource version conflict") + } + + logger.Error(err, "Failed to update GPU status") + return nil, status.Errorf(codes.Internal, "failed to update status: %v", err) + } + + logger.V(1).Info("GPU status updated", "resourceVersion", gpu.GetMetadata().GetResourceVersion()) + + return &v1alpha1.UpdateGpuStatusResponse{Gpu: gpu}, nil +} + +// DeleteGpu removes a GPU from the server. +// +// After deletion, the GPU will no longer appear in ListGpus or GetGpu +// responses. Active WatchGpus streams will receive a DELETED event. +// +// This operation acquires a write lock, blocking all consumer reads. +func (s *GpuService) DeleteGpu(ctx context.Context, req *v1alpha1.DeleteGpuRequest) (*emptypb.Empty, error) { + logger := klog.FromContext(ctx).WithValues( + "method", "DeleteGpu", + "gpuName", req.GetName(), + ) + + // Validate request + if req.GetName() == "" { + logger.V(1).Info("Invalid request: empty name") + return nil, status.Error(codes.InvalidArgument, "name is required") + } + + // Delete GPU (acquires write lock, blocks consumers) + err := s.cache.Delete(req.GetName()) + if err != nil { + if errors.Is(err, cache.ErrGpuNotFound) { + logger.V(1).Info("GPU not found") + return nil, status.Error(codes.NotFound, "gpu not found") + } + + logger.Error(err, "Failed to delete GPU") + return nil, status.Errorf(codes.Internal, "failed to delete gpu: %v", err) + } + + logger.Info("GPU deleted") + + return &emptypb.Empty{}, nil +} + +// Compile-time check that GpuService implements GpuServiceServer. +var _ v1alpha1.GpuServiceServer = (*GpuService)(nil) diff --git a/pkg/deviceapiserver/service/integration_test.go b/pkg/deviceapiserver/service/integration_test.go new file mode 100644 index 000000000..174e63fa5 --- /dev/null +++ b/pkg/deviceapiserver/service/integration_test.go @@ -0,0 +1,610 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service + +import ( + "context" + "sync" + "testing" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "k8s.io/klog/v2" + + v1alpha1 "github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1" + "github.com/nvidia/nvsentinel/pkg/deviceapiserver/cache" +) + +// TestIntegration_FullFlow tests the complete Create -> Watch -> Update -> Delete flow: +// 1. Create GPU +// 2. Watch receives ADDED event +// 3. Update status +// 4. Watch receives MODIFIED event +// 5. Delete GPU +// 6. Watch receives DELETED event +func TestIntegration_FullFlow(t *testing.T) { + logger := klog.Background() + gpuCache := cache.New(logger, nil) + gpuService := NewGpuService(gpuCache) + ctx := context.Background() + + // Start watching before any GPUs exist + stream := newMockWatchStream() + var watchErr error + watchDone := make(chan struct{}) + go func() { + watchErr = gpuService.WatchGpus(&v1alpha1.WatchGpusRequest{}, stream) + close(watchDone) + }() + + // Give watch time to start + time.Sleep(50 * time.Millisecond) + + // Step 1: Create GPU + t.Log("Step 1: Create GPU") + createResp, err := gpuService.CreateGpu(ctx, &v1alpha1.CreateGpuRequest{ + Gpu: &v1alpha1.Gpu{ + Metadata: &v1alpha1.ObjectMeta{Name: "gpu-0"}, + Spec: &v1alpha1.GpuSpec{Uuid: "GPU-1234"}, + }, + }) + if err != nil { + t.Fatalf("CreateGpu failed: %v", err) + } + if !createResp.Created { + t.Error("Expected Created=true") + } + + // Wait for event + time.Sleep(50 * time.Millisecond) + + // Step 2: Verify watch received ADDED event + t.Log("Step 2: Verify watch received ADDED event") + events := stream.getEvents() + if len(events) != 1 { + t.Errorf("Expected 1 event (ADDED), got %d", len(events)) + } else if events[0].Type != cache.EventTypeAdded { + t.Errorf("Expected ADDED event, got %s", events[0].Type) + } else if events[0].Object.GetMetadata().GetName() != "gpu-0" { + t.Errorf("Expected gpu-0, got %s", events[0].Object.GetMetadata().GetName()) + } + + // Step 3: Update status + t.Log("Step 3: Update status") + _, err = gpuService.UpdateGpuStatus(ctx, &v1alpha1.UpdateGpuStatusRequest{ + Name: "gpu-0", + Status: &v1alpha1.GpuStatus{ + Conditions: []*v1alpha1.Condition{ + {Type: "Ready", Status: "True", Message: "GPU is ready"}, + }, + }, + }) + if err != nil { + t.Fatalf("UpdateGpuStatus failed: %v", err) + } + + // Wait for event + time.Sleep(50 * time.Millisecond) + + // Step 4: Verify watch received MODIFIED event + t.Log("Step 4: Verify watch received MODIFIED event") + events = stream.getEvents() + if len(events) != 2 { + t.Errorf("Expected 2 events (ADDED + MODIFIED), got %d", len(events)) + } else if events[1].Type != cache.EventTypeModified { + t.Errorf("Expected MODIFIED event, got %s", events[1].Type) + } + + // Verify condition through GetGpu + getResp, err := gpuService.GetGpu(ctx, &v1alpha1.GetGpuRequest{Name: "gpu-0"}) + if err != nil { + t.Fatalf("GetGpu failed: %v", err) + } + if len(getResp.Gpu.Status.Conditions) != 1 { + t.Errorf("Expected 1 condition, got %d", len(getResp.Gpu.Status.Conditions)) + } else if getResp.Gpu.Status.Conditions[0].Status != "True" { + t.Errorf("Expected status=True, got %s", getResp.Gpu.Status.Conditions[0].Status) + } + + // Step 5: Delete GPU + t.Log("Step 5: Delete GPU") + _, err = gpuService.DeleteGpu(ctx, &v1alpha1.DeleteGpuRequest{Name: "gpu-0"}) + if err != nil { + t.Fatalf("DeleteGpu failed: %v", err) + } + + // Wait for event + time.Sleep(50 * time.Millisecond) + + // Step 6: Verify watch received DELETED event + t.Log("Step 6: Verify watch received DELETED event") + events = stream.getEvents() + if len(events) != 3 { + t.Errorf("Expected 3 events (ADDED + MODIFIED + DELETED), got %d", len(events)) + } else if events[2].Type != cache.EventTypeDeleted { + t.Errorf("Expected DELETED event, got %s", events[2].Type) + } + + // Verify GPU is gone via ListGpus + listResp, err := gpuService.ListGpus(ctx, &v1alpha1.ListGpusRequest{}) + if err != nil { + t.Fatalf("ListGpus failed: %v", err) + } + if len(listResp.GpuList.Items) != 0 { + t.Errorf("Expected 0 GPUs, got %d", len(listResp.GpuList.Items)) + } + + // Cleanup + stream.cancel() + <-watchDone + + if watchErr != nil { + t.Errorf("Watch error: %v", watchErr) + } +} + +// TestIntegration_IdempotentCreate tests that CreateGpu is idempotent +func TestIntegration_IdempotentCreate(t *testing.T) { + logger := klog.Background() + gpuCache := cache.New(logger, nil) + gpuService := NewGpuService(gpuCache) + ctx := context.Background() + + // First create + createResp1, err := gpuService.CreateGpu(ctx, &v1alpha1.CreateGpuRequest{ + Gpu: &v1alpha1.Gpu{ + Metadata: &v1alpha1.ObjectMeta{Name: "gpu-0"}, + Spec: &v1alpha1.GpuSpec{Uuid: "GPU-1234"}, + }, + }) + if err != nil { + t.Fatalf("First CreateGpu failed: %v", err) + } + if !createResp1.Created { + t.Error("Expected Created=true for first create") + } + + // Second create (should return existing without error) + createResp2, err := gpuService.CreateGpu(ctx, &v1alpha1.CreateGpuRequest{ + Gpu: &v1alpha1.Gpu{ + Metadata: &v1alpha1.ObjectMeta{Name: "gpu-0"}, + Spec: &v1alpha1.GpuSpec{Uuid: "GPU-1234"}, + }, + }) + if err != nil { + t.Fatalf("Second CreateGpu failed: %v", err) + } + if createResp2.Created { + t.Error("Expected Created=false for second create") + } + + // Verify we still have exactly one GPU + listResp, err := gpuService.ListGpus(ctx, &v1alpha1.ListGpusRequest{}) + if err != nil { + t.Fatalf("ListGpus failed: %v", err) + } + if len(listResp.GpuList.Items) != 1 { + t.Errorf("Expected 1 GPU, got %d", len(listResp.GpuList.Items)) + } +} + +// TestIntegration_ConcurrentReadersAndWriters tests concurrent access patterns +func TestIntegration_ConcurrentReadersAndWriters(t *testing.T) { + logger := klog.Background() + gpuCache := cache.New(logger, nil) + gpuService := NewGpuService(gpuCache) + ctx := context.Background() + + // Create some GPUs + for i := 0; i < 5; i++ { + _, err := gpuService.CreateGpu(ctx, &v1alpha1.CreateGpuRequest{ + Gpu: &v1alpha1.Gpu{ + Metadata: &v1alpha1.ObjectMeta{Name: "gpu-" + string(rune('0'+i))}, + Spec: &v1alpha1.GpuSpec{Uuid: "GPU"}, + }, + }) + if err != nil { + t.Fatalf("CreateGpu failed: %v", err) + } + } + + var wg sync.WaitGroup + errors := make(chan error, 200) + + // Start multiple concurrent writers updating status + for w := 0; w < 5; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 20; i++ { + gpuName := "gpu-" + string(rune('0'+(i%5))) + _, err := gpuService.UpdateGpuStatus(ctx, &v1alpha1.UpdateGpuStatusRequest{ + Name: gpuName, + Status: &v1alpha1.GpuStatus{ + Conditions: []*v1alpha1.Condition{ + {Type: "TestCondition", Status: "True"}, + }, + }, + }) + if err != nil { + errors <- err + } + } + }() + } + + // Start multiple concurrent readers + for r := 0; r < 10; r++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 50; i++ { + _, err := gpuService.ListGpus(ctx, &v1alpha1.ListGpusRequest{}) + if err != nil { + errors <- err + } + gpuName := "gpu-" + string(rune('0'+(i%5))) + _, _ = gpuService.GetGpu(ctx, &v1alpha1.GetGpuRequest{Name: gpuName}) + } + }() + } + + wg.Wait() + close(errors) + + errorCount := 0 + for err := range errors { + t.Errorf("Concurrent operation error: %v", err) + errorCount++ + } + + if errorCount > 0 { + t.Errorf("Total errors: %d", errorCount) + } +} + +// TestIntegration_WatchWithHighUpdateRate tests watch under high update rates +func TestIntegration_WatchWithHighUpdateRate(t *testing.T) { + logger := klog.Background() + gpuCache := cache.New(logger, nil) + gpuService := NewGpuService(gpuCache) + ctx := context.Background() + + // Create a GPU + _, err := gpuService.CreateGpu(ctx, &v1alpha1.CreateGpuRequest{ + Gpu: &v1alpha1.Gpu{ + Metadata: &v1alpha1.ObjectMeta{Name: "gpu-0"}, + Spec: &v1alpha1.GpuSpec{Uuid: "GPU-0"}, + }, + }) + if err != nil { + t.Fatalf("CreateGpu failed: %v", err) + } + + // Start watching + stream := newMockWatchStream() + watchDone := make(chan struct{}) + go func() { + _ = gpuService.WatchGpus(&v1alpha1.WatchGpusRequest{}, stream) + close(watchDone) + }() + + // Wait for watch to start + time.Sleep(50 * time.Millisecond) + + // Send rapid updates + const numUpdates = 100 + for i := 0; i < numUpdates; i++ { + _, err := gpuService.UpdateGpuStatus(ctx, &v1alpha1.UpdateGpuStatusRequest{ + Name: "gpu-0", + Status: &v1alpha1.GpuStatus{ + Conditions: []*v1alpha1.Condition{ + {Type: "TestCondition", Status: "True", Message: "Update " + string(rune('0'+i%10))}, + }, + }, + }) + if err != nil { + t.Fatalf("UpdateGpuStatus failed at %d: %v", i, err) + } + } + + // Wait for events to propagate + time.Sleep(100 * time.Millisecond) + + // Cancel watch + stream.cancel() + <-watchDone + + events := stream.getEvents() + + // Should have at least 1 initial ADDED + some MODIFIED events + // Note: Some events may be dropped if buffer is full (expected behavior) + if len(events) < 2 { + t.Errorf("Expected at least 2 events, got %d", len(events)) + } + + t.Logf("Received %d events out of %d updates", len(events)-1, numUpdates) +} + +// TestIntegration_MultipleGPUs tests managing multiple GPUs +func TestIntegration_MultipleGPUs(t *testing.T) { + logger := klog.Background() + gpuCache := cache.New(logger, nil) + gpuService := NewGpuService(gpuCache) + ctx := context.Background() + + const numGPUs = 8 // Typical 8-GPU node + + // Create all GPUs + t.Logf("Creating %d GPUs", numGPUs) + for i := 0; i < numGPUs; i++ { + name := "gpu-" + string(rune('0'+i)) + _, err := gpuService.CreateGpu(ctx, &v1alpha1.CreateGpuRequest{ + Gpu: &v1alpha1.Gpu{ + Metadata: &v1alpha1.ObjectMeta{Name: name}, + Spec: &v1alpha1.GpuSpec{Uuid: "GPU-UUID-" + string(rune('0'+i))}, + Status: &v1alpha1.GpuStatus{ + Conditions: []*v1alpha1.Condition{ + {Type: "Ready", Status: "True"}, + }, + }, + }, + }) + if err != nil { + t.Fatalf("CreateGpu %s failed: %v", name, err) + } + } + + // Verify via ListGpus + resp, err := gpuService.ListGpus(ctx, &v1alpha1.ListGpusRequest{}) + if err != nil { + t.Fatalf("ListGpus failed: %v", err) + } + if len(resp.GpuList.Items) != numGPUs { + t.Errorf("Expected %d GPUs, got %d", numGPUs, len(resp.GpuList.Items)) + } + + // Mark one GPU as unhealthy (simulating XID error) + t.Log("Marking gpu-3 as unhealthy (XID error)") + _, err = gpuService.UpdateGpuStatus(ctx, &v1alpha1.UpdateGpuStatusRequest{ + Name: "gpu-3", + Status: &v1alpha1.GpuStatus{ + Conditions: []*v1alpha1.Condition{ + {Type: "Ready", Status: "False", Message: "XID 79 - GPU has fallen off the bus"}, + }, + }, + }) + if err != nil { + t.Fatalf("UpdateGpuStatus failed: %v", err) + } + + // Verify mixed health states + healthyCount := 0 + unhealthyCount := 0 + for i := 0; i < numGPUs; i++ { + name := "gpu-" + string(rune('0'+i)) + gpu, _ := gpuCache.Get(name) + for _, c := range gpu.Status.Conditions { + if c.Type == "Ready" { + if c.Status == "True" { + healthyCount++ + } else { + unhealthyCount++ + } + } + } + } + + if healthyCount != numGPUs-1 { + t.Errorf("Expected %d healthy GPUs, got %d", numGPUs-1, healthyCount) + } + if unhealthyCount != 1 { + t.Errorf("Expected 1 unhealthy GPU, got %d", unhealthyCount) + } + + t.Logf("Multi-GPU test completed: %d healthy, %d unhealthy", healthyCount, unhealthyCount) +} + +// TestIntegration_OptimisticConcurrency tests resource version conflict detection +func TestIntegration_OptimisticConcurrency(t *testing.T) { + logger := klog.Background() + gpuCache := cache.New(logger, nil) + gpuService := NewGpuService(gpuCache) + ctx := context.Background() + + // Create a GPU + createResp, err := gpuService.CreateGpu(ctx, &v1alpha1.CreateGpuRequest{ + Gpu: &v1alpha1.Gpu{ + Metadata: &v1alpha1.ObjectMeta{Name: "gpu-0"}, + Spec: &v1alpha1.GpuSpec{Uuid: "GPU-1234"}, + }, + }) + if err != nil { + t.Fatalf("CreateGpu failed: %v", err) + } + + initialVersion := createResp.Gpu.GetMetadata().GetResourceVersion() + + // Update status with correct version + updateResp, err := gpuService.UpdateGpuStatus(ctx, &v1alpha1.UpdateGpuStatusRequest{ + Name: "gpu-0", + ResourceVersion: initialVersion, + Status: &v1alpha1.GpuStatus{ + Conditions: []*v1alpha1.Condition{ + {Type: "Ready", Status: "True"}, + }, + }, + }) + if err != nil { + t.Fatalf("UpdateGpuStatus with correct version failed: %v", err) + } + + // Try to update with stale version (should fail) + _, err = gpuService.UpdateGpuStatus(ctx, &v1alpha1.UpdateGpuStatusRequest{ + Name: "gpu-0", + ResourceVersion: initialVersion, // This is now stale + Status: &v1alpha1.GpuStatus{ + Conditions: []*v1alpha1.Condition{ + {Type: "Ready", Status: "False"}, + }, + }, + }) + if err == nil { + t.Error("Expected conflict error with stale version, got nil") + } + + // Verify the status was not changed by the conflicting update + getResp, err := gpuService.GetGpu(ctx, &v1alpha1.GetGpuRequest{Name: "gpu-0"}) + if err != nil { + t.Fatalf("GetGpu failed: %v", err) + } + if getResp.Gpu.Status.Conditions[0].Status != "True" { + t.Errorf("Expected status=True (unchanged), got %s", getResp.Gpu.Status.Conditions[0].Status) + } + if getResp.Gpu.GetMetadata().GetResourceVersion() != updateResp.Gpu.GetMetadata().GetResourceVersion() { + t.Errorf("Version should not have changed") + } +} + +// TestUpdateGpu_InvalidResourceVersion verifies that a non-numeric +// resource_version in UpdateGpu returns InvalidArgument instead of silently +// bypassing optimistic concurrency checks. +func TestUpdateGpu_InvalidResourceVersion(t *testing.T) { + gpuCache := cache.New(klog.Background(), nil) + svc := NewGpuService(gpuCache) + ctx := context.Background() + + // Create a GPU first + created, err := svc.CreateGpu(ctx, &v1alpha1.CreateGpuRequest{ + Gpu: &v1alpha1.Gpu{ + Metadata: &v1alpha1.ObjectMeta{Name: "gpu-0"}, + Spec: &v1alpha1.GpuSpec{}, + }, + }) + if err != nil { + t.Fatalf("CreateGpu failed: %v", err) + } + if created.Gpu == nil { + t.Fatal("CreateGpu returned nil gpu") + } + + // Try to update with malformed resource_version + _, err = svc.UpdateGpu(ctx, &v1alpha1.UpdateGpuRequest{ + Gpu: &v1alpha1.Gpu{ + Metadata: &v1alpha1.ObjectMeta{ + Name: "gpu-0", + ResourceVersion: "not-a-number", + }, + Spec: &v1alpha1.GpuSpec{}, + }, + }) + if err == nil { + t.Fatal("Expected error for malformed resource_version, got nil") + } + + st, ok := status.FromError(err) + if !ok { + t.Fatalf("Expected gRPC status error, got: %v", err) + } + if st.Code() != codes.InvalidArgument { + t.Errorf("Expected InvalidArgument, got %s: %s", st.Code(), st.Message()) + } +} + +// TestUpdateGpuStatus_InvalidResourceVersion verifies that a non-numeric +// resource_version in UpdateGpuStatus returns InvalidArgument instead of +// silently bypassing optimistic concurrency checks. +func TestUpdateGpuStatus_InvalidResourceVersion(t *testing.T) { + gpuCache := cache.New(klog.Background(), nil) + svc := NewGpuService(gpuCache) + ctx := context.Background() + + // Create a GPU first + _, err := svc.CreateGpu(ctx, &v1alpha1.CreateGpuRequest{ + Gpu: &v1alpha1.Gpu{ + Metadata: &v1alpha1.ObjectMeta{Name: "gpu-0"}, + Spec: &v1alpha1.GpuSpec{}, + }, + }) + if err != nil { + t.Fatalf("CreateGpu failed: %v", err) + } + + // Try to update status with malformed resource_version + _, err = svc.UpdateGpuStatus(ctx, &v1alpha1.UpdateGpuStatusRequest{ + Name: "gpu-0", + ResourceVersion: "not-a-number", + Status: &v1alpha1.GpuStatus{ + Conditions: []*v1alpha1.Condition{ + {Type: "Ready", Status: "True"}, + }, + }, + }) + if err == nil { + t.Fatal("Expected error for malformed resource_version, got nil") + } + + st, ok := status.FromError(err) + if !ok { + t.Fatalf("Expected gRPC status error, got: %v", err) + } + if st.Code() != codes.InvalidArgument { + t.Errorf("Expected InvalidArgument, got %s: %s", st.Code(), st.Message()) + } +} + +// mockWatchStream implements grpc.ServerStreamingServer for testing +type mockWatchStream struct { + ctx context.Context + cancel context.CancelFunc + events []*v1alpha1.WatchGpusResponse + mu sync.Mutex +} + +func newMockWatchStream() *mockWatchStream { + ctx, cancel := context.WithCancel(context.Background()) + return &mockWatchStream{ + ctx: ctx, + cancel: cancel, + } +} + +func (m *mockWatchStream) Send(resp *v1alpha1.WatchGpusResponse) error { + m.mu.Lock() + defer m.mu.Unlock() + m.events = append(m.events, resp) + return nil +} + +func (m *mockWatchStream) Context() context.Context { + return m.ctx +} + +func (m *mockWatchStream) getEvents() []*v1alpha1.WatchGpusResponse { + m.mu.Lock() + defer m.mu.Unlock() + result := make([]*v1alpha1.WatchGpusResponse, len(m.events)) + copy(result, m.events) + return result +} + +// These methods are required by the grpc.ServerStreamingServer interface +func (m *mockWatchStream) SetHeader(md metadata.MD) error { return nil } +func (m *mockWatchStream) SendHeader(md metadata.MD) error { return nil } +func (m *mockWatchStream) SetTrailer(md metadata.MD) {} +func (m *mockWatchStream) SendMsg(msg interface{}) error { return nil } +func (m *mockWatchStream) RecvMsg(msg interface{}) error { return nil } diff --git a/pkg/healthprovider/nvml_source.go b/pkg/healthprovider/nvml_source.go new file mode 100644 index 000000000..523af3fcc --- /dev/null +++ b/pkg/healthprovider/nvml_source.go @@ -0,0 +1,498 @@ +//go:build nvml + +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package healthprovider + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "github.com/NVIDIA/go-nvml/pkg/nvml" + "k8s.io/klog/v2" +) + +const ( + // NVMLSourceName is the identifier for the NVML health source. + NVMLSourceName = "nvml" + + // EventTimeout is the timeout for NVML event wait (milliseconds). + EventTimeout = 5000 +) + +// NVMLSourceConfig holds configuration for the NVML health source. +type NVMLSourceConfig struct { + // DriverRoot is the path where NVIDIA driver libraries are located. + DriverRoot string + + // IgnoredXids is a list of XID error codes to ignore. + IgnoredXids []uint64 + + // HealthCheckEnabled enables XID event monitoring. + HealthCheckEnabled bool +} + +// DefaultNVMLSourceConfig returns default NVML source configuration. +func DefaultNVMLSourceConfig() NVMLSourceConfig { + return NVMLSourceConfig{ + DriverRoot: "/run/nvidia/driver", + IgnoredXids: nil, + HealthCheckEnabled: true, + } +} + +// NVMLSource implements HealthSource for NVML-based GPU monitoring. +type NVMLSource struct { + config NVMLSourceConfig + logger klog.Logger + handler HealthEventHandler + + nvmllib nvml.Interface + eventSet nvml.EventSet + + mu sync.Mutex + initialized bool + running bool + gpuUUIDs []string + ignoredXidsMap map[uint64]bool + + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup +} + +// NewNVMLSource creates a new NVML health source. +func NewNVMLSource(cfg NVMLSourceConfig, logger klog.Logger) *NVMLSource { + // Build ignored XIDs map + ignoredMap := make(map[uint64]bool) + for _, xid := range DefaultIgnoredXids { + ignoredMap[xid] = true + } + for _, xid := range cfg.IgnoredXids { + ignoredMap[xid] = true + } + + return &NVMLSource{ + config: cfg, + logger: logger.WithName("nvml-source"), + ignoredXidsMap: ignoredMap, + } +} + +// Name returns the source identifier. +func (s *NVMLSource) Name() string { + return NVMLSourceName +} + +// Start initializes NVML and begins health monitoring. +func (s *NVMLSource) Start(ctx context.Context, handler HealthEventHandler) error { + s.mu.Lock() + defer s.mu.Unlock() + + if s.initialized { + return fmt.Errorf("NVML source already started") + } + + s.handler = handler + s.ctx, s.cancel = context.WithCancel(ctx) + + // Initialize NVML + if err := s.initNVML(); err != nil { + return err + } + + // Enumerate and register GPUs + if err := s.enumerateGPUs(); err != nil { + s.shutdownNVML() + return err + } + + // Start health monitoring if enabled and GPUs are present + if s.config.HealthCheckEnabled && len(s.gpuUUIDs) > 0 { + s.wg.Add(1) + go s.runHealthMonitor() + } + + s.initialized = true + s.running = true + s.logger.Info("NVML source started", "gpuCount", len(s.gpuUUIDs)) + + return nil +} + +// Stop shuts down the NVML source. +func (s *NVMLSource) Stop() { + s.mu.Lock() + defer s.mu.Unlock() + + if !s.initialized { + return + } + + s.logger.Info("Stopping NVML source") + + if s.cancel != nil { + s.cancel() + } + + s.wg.Wait() + s.shutdownNVML() + + s.initialized = false + s.running = false + s.logger.Info("NVML source stopped") +} + +// IsRunning returns true if the source is actively monitoring. +func (s *NVMLSource) IsRunning() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.running +} + +// initNVML initializes the NVML library. +func (s *NVMLSource) initNVML() error { + libraryPath := s.findDriverLibrary() + if libraryPath != "" { + s.logger.V(2).Info("Using NVML library", "path", libraryPath) + s.nvmllib = nvml.New(nvml.WithLibraryPath(libraryPath)) + } else { + s.logger.V(2).Info("Using system default NVML library") + s.nvmllib = nvml.New() + } + + ret := s.nvmllib.Init() + if ret != nvml.SUCCESS { + return fmt.Errorf("NVML init failed: %v", nvml.ErrorString(ret)) + } + + if version, ret := s.nvmllib.SystemGetDriverVersion(); ret == nvml.SUCCESS { + s.logger.Info("NVML initialized", "driverVersion", version) + } + + return nil +} + +// shutdownNVML shuts down the NVML library. +func (s *NVMLSource) shutdownNVML() { + if s.eventSet != nil { + s.eventSet.Free() + s.eventSet = nil + } + + if s.nvmllib != nil { + s.nvmllib.Shutdown() + } +} + +// findDriverLibrary locates the NVML library. +func (s *NVMLSource) findDriverLibrary() string { + if s.config.DriverRoot == "" { + return "" + } + + paths := []string{ + filepath.Join(s.config.DriverRoot, "usr/lib64/libnvidia-ml.so.1"), + filepath.Join(s.config.DriverRoot, "usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1"), + filepath.Join(s.config.DriverRoot, "usr/lib/libnvidia-ml.so.1"), + filepath.Join(s.config.DriverRoot, "lib64/libnvidia-ml.so.1"), + filepath.Join(s.config.DriverRoot, "lib/libnvidia-ml.so.1"), + } + + for _, path := range paths { + if _, err := os.Stat(path); err == nil { + return path + } + } + + return "" +} + +// enumerateGPUs discovers GPUs and registers them with the handler. +func (s *NVMLSource) enumerateGPUs() error { + count, ret := s.nvmllib.DeviceGetCount() + if ret != nvml.SUCCESS { + return fmt.Errorf("failed to get device count: %v", nvml.ErrorString(ret)) + } + + if count == 0 { + s.logger.Info("No GPUs found on this node") + return nil + } + + s.logger.Info("Enumerating GPUs", "count", count) + s.gpuUUIDs = make([]string, 0, count) + + for i := 0; i < count; i++ { + device, ret := s.nvmllib.DeviceGetHandleByIndex(i) + if ret != nvml.SUCCESS { + s.logger.Error(nil, "Failed to get device handle", "index", i, "error", nvml.ErrorString(ret)) + continue + } + + uuid, ret := device.GetUUID() + if ret != nvml.SUCCESS { + s.logger.Error(nil, "Failed to get device UUID", "index", i, "error", nvml.ErrorString(ret)) + continue + } + + productName, _ := device.GetName() + var memoryBytes uint64 + if memInfo, ret := device.GetMemoryInfo(); ret == nvml.SUCCESS { + memoryBytes = memInfo.Total + } + + gpu := &GPUInfo{ + UUID: uuid, + ProductName: productName, + MemoryBytes: memoryBytes, + Index: i, + Source: NVMLSourceName, + } + + if err := s.handler.OnGPUDiscovered(s.ctx, gpu); err != nil { + s.logger.Error(err, "Failed to register GPU", "uuid", uuid) + continue + } + + s.gpuUUIDs = append(s.gpuUUIDs, uuid) + s.logger.Info("GPU enumerated", + "uuid", uuid, + "productName", productName, + "memory", formatBytes(memoryBytes), + ) + } + + s.logger.Info("GPU enumeration complete", "registered", len(s.gpuUUIDs)) + return nil +} + +// runHealthMonitor monitors NVML events for GPU health changes. +func (s *NVMLSource) runHealthMonitor() { + defer s.wg.Done() + + // Create event set + eventSet, ret := s.nvmllib.EventSetCreate() + if ret != nvml.SUCCESS { + s.logger.Error(nil, "Failed to create event set", "error", nvml.ErrorString(ret)) + return + } + s.eventSet = eventSet + + // Register devices for XID events + deviceCount, ret := s.nvmllib.DeviceGetCount() + if ret != nvml.SUCCESS { + s.logger.Error(nil, "Failed to get device count", "error", nvml.ErrorString(ret)) + return + } + + registeredDevices := 0 + for i := 0; i < deviceCount; i++ { + device, ret := s.nvmllib.DeviceGetHandleByIndex(i) + if ret != nvml.SUCCESS { + continue + } + + // Check supported event types + supportedTypes, ret := device.GetSupportedEventTypes() + if ret != nvml.SUCCESS { + s.logger.V(2).Info("Cannot get supported event types", "index", i) + continue + } + + // Register for all supported error event types + eventTypes := uint64(0) + if supportedTypes&nvml.EventTypeXidCriticalError != 0 { + eventTypes |= nvml.EventTypeXidCriticalError + } + if supportedTypes&nvml.EventTypeSingleBitEccError != 0 { + eventTypes |= nvml.EventTypeSingleBitEccError + } + if supportedTypes&nvml.EventTypeDoubleBitEccError != 0 { + eventTypes |= nvml.EventTypeDoubleBitEccError + } + + if eventTypes == 0 { + continue + } + + ret = device.RegisterEvents(eventTypes, eventSet) + if ret != nvml.SUCCESS { + s.logger.V(1).Info("Failed to register events", "index", i, "error", nvml.ErrorString(ret)) + continue + } + registeredDevices++ + } + + s.logger.Info("Health monitor started", "registeredDevices", registeredDevices) + + // Event loop + for { + select { + case <-s.ctx.Done(): + return + default: + } + + data, ret := eventSet.Wait(EventTimeout) + if ret == nvml.ERROR_TIMEOUT { + continue + } + if ret != nvml.SUCCESS { + s.logger.V(2).Info("Event wait error", "error", nvml.ErrorString(ret)) + continue + } + + s.handleXIDEvent(data) + } +} + +// handleXIDEvent processes an XID error event. +func (s *NVMLSource) handleXIDEvent(data nvml.EventData) { + uuid, ret := data.Device.GetUUID() + if ret != nvml.SUCCESS { + s.logger.Error(nil, "Failed to get device UUID from event") + return + } + + xid := data.EventData + eventType := data.EventType + + s.logger.Info("XID event received", + "uuid", uuid, + "xid", xid, + "eventType", eventType, + ) + + // Check if XID should be ignored + if s.isIgnoredXid(xid) { + s.logger.V(1).Info("Ignoring XID event", "uuid", uuid, "xid", xid) + return + } + + // Determine severity + isFatal := IsCriticalXid(xid) + isHealthy := !isFatal + + event := &HealthEvent{ + GPUUID: uuid, + Source: NVMLSourceName, + IsHealthy: isHealthy, + IsFatal: isFatal, + ErrorCodes: []string{fmt.Sprintf("%d", xid)}, + Reason: xidToReason(xid), + Message: xidToMessage(xid), + DetectedAt: time.Now(), + } + + if err := s.handler.OnGPUHealthChanged(s.ctx, event); err != nil { + s.logger.Error(err, "Failed to report health event", "uuid", uuid) + } +} + +// isIgnoredXid checks if an XID should be ignored. +func (s *NVMLSource) isIgnoredXid(xid uint64) bool { + return s.ignoredXidsMap[xid] +} + +// GPUUUIDs returns the list of discovered GPU UUIDs. +func (s *NVMLSource) GPUUUIDs() []string { + s.mu.Lock() + defer s.mu.Unlock() + result := make([]string, len(s.gpuUUIDs)) + copy(result, s.gpuUUIDs) + return result +} + +// XID classification utilities + +// DefaultIgnoredXids are XID codes that are typically application errors. +var DefaultIgnoredXids = []uint64{ + 13, // Graphics Engine Exception + 31, // GPU memory page fault + 43, // GPU stopped processing + 45, // Preemptive cleanup, due to previous errors +} + +// CriticalXids are XID codes that indicate hardware failures. +var CriticalXids = map[uint64]bool{ + 48: true, // Double bit ECC error + 63: true, // ECC page retirement or row remapping recording event + 64: true, // ECC page retirement or row remapping threshold exceeded + 74: true, // NVLink error + 79: true, // GPU has fallen off the bus + 92: true, // High single-bit ECC error rate + 94: true, // Contained ECC error + 95: true, // Uncontained ECC error + 119: true, // GSP error + 120: true, // GSP error +} + +// IsCriticalXid returns true if the XID indicates a critical hardware failure. +func IsCriticalXid(xid uint64) bool { + return CriticalXids[xid] +} + +func xidToReason(xid uint64) string { + if IsCriticalXid(xid) { + return "CriticalXIDError" + } + return "XIDError" +} + +func xidToMessage(xid uint64) string { + descriptions := map[uint64]string{ + 13: "Graphics Engine Exception", + 31: "GPU memory page fault", + 43: "GPU stopped processing", + 45: "Preemptive cleanup due to previous errors", + 48: "Double bit ECC error", + 63: "ECC page retirement recording event", + 64: "ECC page retirement threshold exceeded", + 74: "NVLink error", + 79: "GPU has fallen off the bus", + 92: "High single-bit ECC error rate", + 94: "Contained ECC error", + 95: "Uncontained ECC error", + 119: "GSP RPC timeout", + 120: "GSP error", + } + + if desc, ok := descriptions[xid]; ok { + return fmt.Sprintf("XID %d: %s", xid, desc) + } + return fmt.Sprintf("XID %d error", xid) +} + +func formatBytes(bytes uint64) string { + const GB = 1024 * 1024 * 1024 + const MB = 1024 * 1024 + const KB = 1024 + + switch { + case bytes >= GB: + return fmt.Sprintf("%.1f GB", float64(bytes)/float64(GB)) + case bytes >= MB: + return fmt.Sprintf("%.1f MB", float64(bytes)/float64(MB)) + case bytes >= KB: + return fmt.Sprintf("%.1f KB", float64(bytes)/float64(KB)) + default: + return fmt.Sprintf("%d B", bytes) + } +} diff --git a/pkg/healthprovider/nvml_source_stub.go b/pkg/healthprovider/nvml_source_stub.go new file mode 100644 index 000000000..8e22e5e1a --- /dev/null +++ b/pkg/healthprovider/nvml_source_stub.go @@ -0,0 +1,86 @@ +//go:build !nvml + +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package healthprovider + +import ( + "context" + "fmt" + + "k8s.io/klog/v2" +) + +const NVMLSourceName = "nvml" + +// NVMLSourceConfig holds configuration for the NVML health source. +type NVMLSourceConfig struct { + DriverRoot string + IgnoredXids []uint64 + HealthCheckEnabled bool +} + +// DefaultNVMLSourceConfig returns default NVML source configuration. +func DefaultNVMLSourceConfig() NVMLSourceConfig { + return NVMLSourceConfig{ + DriverRoot: "/run/nvidia/driver", + HealthCheckEnabled: true, + } +} + +// NVMLSource is a stub implementation when NVML is not available. +type NVMLSource struct { + logger klog.Logger +} + +// NewNVMLSource creates a stub NVML source. +func NewNVMLSource(_ NVMLSourceConfig, logger klog.Logger) *NVMLSource { + return &NVMLSource{logger: logger.WithName("nvml-source-stub")} +} + +// Name returns the source identifier. +func (s *NVMLSource) Name() string { + return NVMLSourceName +} + +// Start returns an error indicating NVML is not available. +func (s *NVMLSource) Start(_ context.Context, _ HealthEventHandler) error { + s.logger.Info("NVML source not available (build without nvml tag)") + return fmt.Errorf("NVML not available: binary built without nvml tag") +} + +// Stop is a no-op for the stub. +func (s *NVMLSource) Stop() {} + +// IsRunning always returns false for the stub. +func (s *NVMLSource) IsRunning() bool { + return false +} + +// GPUUUIDs returns an empty slice for the stub. +func (s *NVMLSource) GPUUUIDs() []string { + return nil +} + +// IsCriticalXid returns false for the stub. +func IsCriticalXid(_ uint64) bool { + return false +} + +// DefaultIgnoredXids for stub. +var DefaultIgnoredXids = []uint64{} + +// CriticalXids for stub. +var CriticalXids = map[uint64]bool{} diff --git a/pkg/healthprovider/provider.go b/pkg/healthprovider/provider.go new file mode 100644 index 000000000..5ce685e85 --- /dev/null +++ b/pkg/healthprovider/provider.go @@ -0,0 +1,544 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package healthprovider provides a modular GPU health monitoring service. +// +// The HealthProvider coordinates multiple health sources (NVML, syslog, DCGM, CSP) +// and sends health events to the device-api-server via gRPC. +// +// Architecture: +// +// ┌──────────────────────────────────────────────────────┐ +// │ HealthProvider │ +// │ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ +// │ │ NVML │ │ Syslog │ │ DCGM │ │ CSP │ │ +// │ │ Source │ │ Source │ │ Source │ │ Source │ │ +// │ └────┬─────┘ └────┬─────┘ └────┬─────┘ └────┬─────┘ │ +// │ └────────────┴────────────┴────────────┘ │ +// │ │ │ +// │ HealthEventHandler │ +// └────────────────────────┼─────────────────────────────┘ +// │ gRPC +// ▼ +// ┌─────────────────────┐ +// │ device-api-server │ +// └─────────────────────┘ +package healthprovider + +import ( + "context" + "fmt" + "net/http" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/protobuf/types/known/timestamppb" + "k8s.io/klog/v2" + + v1alpha1 "github.com/nvidia/nvsentinel/api/gen/go/device/v1alpha1" +) + +const ( + // DefaultHeartbeatInterval is how often to send heartbeats. + DefaultHeartbeatInterval = 10 * time.Second + + // DefaultHealthCheckPort is the HTTP port for health checks. + DefaultHealthCheckPort = 8082 + + // DefaultServerAddress is the default device-api-server address. + DefaultServerAddress = "localhost:9001" + + // DefaultConnectionRetryInterval is how long to wait between connection attempts. + DefaultConnectionRetryInterval = 5 * time.Second + + // DefaultMaxConnectionRetries is the maximum number of connection attempts. + DefaultMaxConnectionRetries = 60 +) + +// HealthSource is an interface for pluggable health monitoring sources. +type HealthSource interface { + // Name returns the source identifier (e.g., "nvml", "syslog", "dcgm"). + Name() string + + // Start begins health monitoring. Returns an error if initialization fails. + Start(ctx context.Context, handler HealthEventHandler) error + + // Stop stops health monitoring and releases resources. + Stop() + + // IsRunning returns true if the source is actively monitoring. + IsRunning() bool +} + +// HealthEventHandler is the interface for handling health events from sources. +type HealthEventHandler interface { + // OnGPUDiscovered is called when a new GPU is discovered. + OnGPUDiscovered(ctx context.Context, gpu *GPUInfo) error + + // OnGPUHealthChanged is called when GPU health status changes. + OnGPUHealthChanged(ctx context.Context, event *HealthEvent) error + + // OnGPURemoved is called when a GPU is no longer present. + OnGPURemoved(ctx context.Context, gpuUUID string) error +} + +// GPUInfo contains information about a discovered GPU. +type GPUInfo struct { + UUID string + ProductName string + MemoryBytes uint64 + Index int + Source string +} + +// HealthEvent represents a health status change event. +type HealthEvent struct { + GPUUID string + Source string + IsHealthy bool + IsFatal bool + ErrorCodes []string + Reason string + Message string + DetectedAt time.Time +} + +// Config holds configuration for the HealthProvider. +type Config struct { + // ServerAddress is the device-api-server gRPC address. + ServerAddress string + + // ProviderID is the unique identifier for this provider instance. + ProviderID string + + // NodeName is the Kubernetes node name. + NodeName string + + // HealthCheckPort is the HTTP port for liveness/readiness probes. + HealthCheckPort int + + // HeartbeatInterval is how often to verify server connectivity. + HeartbeatInterval time.Duration + + // ConnectionRetryInterval is how long to wait between connection attempts. + ConnectionRetryInterval time.Duration + + // MaxConnectionRetries is the maximum number of connection attempts. + MaxConnectionRetries int +} + +// DefaultConfig returns a Config with sensible defaults. +func DefaultConfig() Config { + return Config{ + ServerAddress: DefaultServerAddress, + ProviderID: "health-provider", + NodeName: "", + HealthCheckPort: DefaultHealthCheckPort, + HeartbeatInterval: DefaultHeartbeatInterval, + ConnectionRetryInterval: DefaultConnectionRetryInterval, + MaxConnectionRetries: DefaultMaxConnectionRetries, + } +} + +// Provider is the main health provider service. +type Provider struct { + config Config + logger klog.Logger + + // Health sources + sources []HealthSource + + // gRPC connection + conn *grpc.ClientConn + gpuClient v1alpha1.GpuServiceClient + healthClient grpc_health_v1.HealthClient + + // State + mu sync.RWMutex + connected bool + healthy bool + gpuUUIDs map[string]bool + sourceInfo map[string]bool // tracks which sources are running + + // Lifecycle + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup +} + +// New creates a new HealthProvider. +func New(cfg Config, logger klog.Logger) *Provider { + return &Provider{ + config: cfg, + logger: logger.WithName("health-provider"), + sources: make([]HealthSource, 0), + gpuUUIDs: make(map[string]bool), + sourceInfo: make(map[string]bool), + } +} + +// RegisterSource adds a health source to the provider. +// Must be called before Start(). +func (p *Provider) RegisterSource(source HealthSource) { + p.sources = append(p.sources, source) + p.logger.Info("Registered health source", "source", source.Name()) +} + +// Run starts the provider and blocks until context is cancelled. +func (p *Provider) Run(ctx context.Context) error { + p.ctx, p.cancel = context.WithCancel(ctx) + defer p.cancel() + + // Start HTTP health server + p.wg.Add(1) + go p.runHealthServer() + + // Connect to device-api-server + if err := p.connectWithRetry(); err != nil { + return fmt.Errorf("failed to connect to server: %w", err) + } + defer p.disconnect() + + // Start all health sources + sourcesStarted := 0 + for _, source := range p.sources { + p.logger.Info("Starting health source", "source", source.Name()) + if err := source.Start(p.ctx, p); err != nil { + p.logger.Error(err, "Failed to start health source", "source", source.Name()) + continue + } + p.sourceInfo[source.Name()] = true + sourcesStarted++ + } + + if sourcesStarted == 0 && len(p.sources) > 0 { + return fmt.Errorf("no health sources could be started") + } + + // Start heartbeat loop + p.wg.Add(1) + go p.runHeartbeatLoop() + + // Mark as healthy + p.setHealthy(true) + p.logger.Info("Health provider running", + "sources", sourcesStarted, + "serverAddress", p.config.ServerAddress, + ) + + // Wait for shutdown + <-p.ctx.Done() + + // Graceful shutdown + p.setHealthy(false) + for _, source := range p.sources { + source.Stop() + } + p.wg.Wait() + + return nil +} + +// connectWithRetry establishes connection to device-api-server with retry logic. +func (p *Provider) connectWithRetry() error { + var lastErr error + + for i := 0; i < p.config.MaxConnectionRetries; i++ { + select { + case <-p.ctx.Done(): + return p.ctx.Err() + default: + } + + conn, err := grpc.NewClient( + p.config.ServerAddress, + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + lastErr = err + p.logger.V(1).Info("Connection attempt failed", + "attempt", i+1, + "error", err, + ) + time.Sleep(p.config.ConnectionRetryInterval) + continue + } + + p.conn = conn + p.gpuClient = v1alpha1.NewGpuServiceClient(conn) + p.healthClient = grpc_health_v1.NewHealthClient(conn) + + // Wait for server to be ready + if err := p.waitForServerReady(); err != nil { + conn.Close() + lastErr = err + p.logger.V(1).Info("Server not ready", + "attempt", i+1, + "error", err, + ) + time.Sleep(p.config.ConnectionRetryInterval) + continue + } + + p.mu.Lock() + p.connected = true + p.mu.Unlock() + + p.logger.Info("Connected to device-api-server", + "address", p.config.ServerAddress, + ) + return nil + } + + return fmt.Errorf("failed to connect after %d attempts: %w", + p.config.MaxConnectionRetries, lastErr) +} + +// waitForServerReady checks if the server is healthy. +func (p *Provider) waitForServerReady() error { + ctx, cancel := context.WithTimeout(p.ctx, 5*time.Second) + defer cancel() + + resp, err := p.healthClient.Check(ctx, &grpc_health_v1.HealthCheckRequest{}) + if err != nil { + return fmt.Errorf("health check failed: %w", err) + } + + if resp.Status != grpc_health_v1.HealthCheckResponse_SERVING { + return fmt.Errorf("server not serving: %v", resp.Status) + } + + return nil +} + +// disconnect closes the gRPC connection. +func (p *Provider) disconnect() { + if p.conn != nil { + p.conn.Close() + p.conn = nil + } + p.mu.Lock() + p.connected = false + p.mu.Unlock() +} + +// HealthEventHandler interface implementation + +// OnGPUDiscovered registers a newly discovered GPU with device-api-server. +func (p *Provider) OnGPUDiscovered(ctx context.Context, gpu *GPUInfo) error { + p.mu.Lock() + defer p.mu.Unlock() + + if p.gpuUUIDs[gpu.UUID] { + // Already registered + return nil + } + + reqCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + req := &v1alpha1.CreateGpuRequest{ + Gpu: &v1alpha1.Gpu{ + Metadata: &v1alpha1.ObjectMeta{Name: gpu.UUID}, + Spec: &v1alpha1.GpuSpec{Uuid: gpu.UUID}, + Status: &v1alpha1.GpuStatus{ + Conditions: []*v1alpha1.Condition{ + { + Type: fmt.Sprintf("%sReady", gpu.Source), + Status: "True", + Reason: "Initialized", + Message: fmt.Sprintf("GPU discovered via %s: %s", gpu.Source, gpu.ProductName), + LastTransitionTime: timestamppb.Now(), + }, + }, + }, + }, + } + + _, err := p.gpuClient.CreateGpu(reqCtx, req) + if err != nil { + return fmt.Errorf("failed to create GPU: %w", err) + } + + p.gpuUUIDs[gpu.UUID] = true + p.logger.Info("GPU registered", + "uuid", gpu.UUID, + "productName", gpu.ProductName, + "source", gpu.Source, + ) + + return nil +} + +// OnGPUHealthChanged updates GPU health status in device-api-server. +func (p *Provider) OnGPUHealthChanged(ctx context.Context, event *HealthEvent) error { + reqCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + status := "True" + if !event.IsHealthy { + status = "False" + } + + req := &v1alpha1.UpdateGpuStatusRequest{ + Name: event.GPUUID, + Status: &v1alpha1.GpuStatus{ + Conditions: []*v1alpha1.Condition{ + { + Type: fmt.Sprintf("%sReady", event.Source), + Status: status, + Reason: event.Reason, + Message: event.Message, + LastTransitionTime: timestamppb.New(event.DetectedAt), + }, + }, + }, + } + + _, err := p.gpuClient.UpdateGpuStatus(reqCtx, req) + if err != nil { + return fmt.Errorf("failed to update GPU status: %w", err) + } + + p.logger.Info("GPU health updated", + "uuid", event.GPUUID, + "healthy", event.IsHealthy, + "fatal", event.IsFatal, + "reason", event.Reason, + "source", event.Source, + ) + + return nil +} + +// OnGPURemoved handles GPU removal. +func (p *Provider) OnGPURemoved(ctx context.Context, gpuUUID string) error { + p.mu.Lock() + defer p.mu.Unlock() + + if !p.gpuUUIDs[gpuUUID] { + return nil + } + + reqCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + _, err := p.gpuClient.DeleteGpu(reqCtx, &v1alpha1.DeleteGpuRequest{Name: gpuUUID}) + if err != nil { + return fmt.Errorf("failed to delete GPU: %w", err) + } + + delete(p.gpuUUIDs, gpuUUID) + p.logger.Info("GPU removed", "uuid", gpuUUID) + + return nil +} + +// runHeartbeatLoop periodically verifies server connectivity. +func (p *Provider) runHeartbeatLoop() { + defer p.wg.Done() + + ticker := time.NewTicker(p.config.HeartbeatInterval) + defer ticker.Stop() + + for { + select { + case <-p.ctx.Done(): + return + case <-ticker.C: + if err := p.waitForServerReady(); err != nil { + p.logger.Error(err, "Heartbeat failed") + } + } + } +} + +// runHealthServer runs the HTTP health check server. +func (p *Provider) runHealthServer() { + defer p.wg.Done() + + mux := http.NewServeMux() + mux.HandleFunc("/healthz", p.handleHealthz) + mux.HandleFunc("/readyz", p.handleReadyz) + mux.HandleFunc("/livez", p.handleHealthz) + + server := &http.Server{ + Addr: fmt.Sprintf(":%d", p.config.HealthCheckPort), + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + } + + go func() { + <-p.ctx.Done() + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + server.Shutdown(shutdownCtx) + }() + + p.logger.Info("Health server started", "port", p.config.HealthCheckPort) + if err := server.ListenAndServe(); err != http.ErrServerClosed { + p.logger.Error(err, "Health server error") + } +} + +func (p *Provider) handleHealthz(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("ok\n")) +} + +func (p *Provider) handleReadyz(w http.ResponseWriter, _ *http.Request) { + p.mu.RLock() + healthy := p.healthy + p.mu.RUnlock() + + if healthy { + w.WriteHeader(http.StatusOK) + w.Write([]byte("ok\n")) + } else { + w.WriteHeader(http.StatusServiceUnavailable) + w.Write([]byte("not ready\n")) + } +} + +func (p *Provider) setHealthy(healthy bool) { + p.mu.Lock() + p.healthy = healthy + p.mu.Unlock() +} + +// IsConnected returns true if connected to device-api-server. +func (p *Provider) IsConnected() bool { + p.mu.RLock() + defer p.mu.RUnlock() + return p.connected +} + +// IsHealthy returns true if the provider is healthy. +func (p *Provider) IsHealthy() bool { + p.mu.RLock() + defer p.mu.RUnlock() + return p.healthy +} + +// GPUCount returns the number of registered GPUs. +func (p *Provider) GPUCount() int { + p.mu.RLock() + defer p.mu.RUnlock() + return len(p.gpuUUIDs) +} diff --git a/pkg/healthprovider/provider_test.go b/pkg/healthprovider/provider_test.go new file mode 100644 index 000000000..a0b45c1b3 --- /dev/null +++ b/pkg/healthprovider/provider_test.go @@ -0,0 +1,214 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package healthprovider + +import ( + "context" + "testing" + "time" + + "k8s.io/klog/v2" +) + +// mockHealthSource is a mock implementation of HealthSource for testing. +type mockHealthSource struct { + name string + running bool + startFn func(ctx context.Context, handler HealthEventHandler) error + stopFn func() +} + +func (m *mockHealthSource) Name() string { + return m.name +} + +func (m *mockHealthSource) Start(ctx context.Context, handler HealthEventHandler) error { + if m.startFn != nil { + return m.startFn(ctx, handler) + } + m.running = true + return nil +} + +func (m *mockHealthSource) Stop() { + if m.stopFn != nil { + m.stopFn() + } + m.running = false +} + +func (m *mockHealthSource) IsRunning() bool { + return m.running +} + +func TestDefaultConfig(t *testing.T) { + cfg := DefaultConfig() + + if cfg.ServerAddress != DefaultServerAddress { + t.Errorf("ServerAddress = %v, want %v", cfg.ServerAddress, DefaultServerAddress) + } + if cfg.HealthCheckPort != DefaultHealthCheckPort { + t.Errorf("HealthCheckPort = %v, want %v", cfg.HealthCheckPort, DefaultHealthCheckPort) + } + if cfg.HeartbeatInterval != DefaultHeartbeatInterval { + t.Errorf("HeartbeatInterval = %v, want %v", cfg.HeartbeatInterval, DefaultHeartbeatInterval) + } +} + +func TestProviderRegisterSource(t *testing.T) { + logger := klog.Background() + provider := New(DefaultConfig(), logger) + + source1 := &mockHealthSource{name: "source1"} + source2 := &mockHealthSource{name: "source2"} + + provider.RegisterSource(source1) + provider.RegisterSource(source2) + + if len(provider.sources) != 2 { + t.Errorf("Expected 2 sources, got %d", len(provider.sources)) + } +} + +func TestProviderIsConnected(t *testing.T) { + logger := klog.Background() + provider := New(DefaultConfig(), logger) + + // Initially not connected + if provider.IsConnected() { + t.Error("Expected not connected initially") + } + + // Simulate connection + provider.mu.Lock() + provider.connected = true + provider.mu.Unlock() + + if !provider.IsConnected() { + t.Error("Expected connected after setting flag") + } +} + +func TestProviderIsHealthy(t *testing.T) { + logger := klog.Background() + provider := New(DefaultConfig(), logger) + + // Initially not healthy + if provider.IsHealthy() { + t.Error("Expected not healthy initially") + } + + // Set healthy + provider.setHealthy(true) + + if !provider.IsHealthy() { + t.Error("Expected healthy after setting") + } +} + +func TestProviderGPUCount(t *testing.T) { + logger := klog.Background() + provider := New(DefaultConfig(), logger) + + if provider.GPUCount() != 0 { + t.Error("Expected 0 GPUs initially") + } + + // Simulate GPU registration + provider.mu.Lock() + provider.gpuUUIDs["GPU-123"] = true + provider.gpuUUIDs["GPU-456"] = true + provider.mu.Unlock() + + if provider.GPUCount() != 2 { + t.Errorf("Expected 2 GPUs, got %d", provider.GPUCount()) + } +} + +func TestGPUInfo(t *testing.T) { + gpu := &GPUInfo{ + UUID: "GPU-12345678", + ProductName: "NVIDIA A100", + MemoryBytes: 40 * 1024 * 1024 * 1024, // 40 GB + Index: 0, + Source: "nvml", + } + + if gpu.UUID != "GPU-12345678" { + t.Errorf("UUID = %v, want GPU-12345678", gpu.UUID) + } + if gpu.Source != "nvml" { + t.Errorf("Source = %v, want nvml", gpu.Source) + } +} + +func TestHealthEvent(t *testing.T) { + event := &HealthEvent{ + GPUUID: "GPU-12345678", + Source: "nvml", + IsHealthy: false, + IsFatal: true, + ErrorCodes: []string{"79"}, + Reason: "CriticalXIDError", + Message: "GPU has fallen off the bus", + DetectedAt: time.Now(), + } + + if event.GPUUID != "GPU-12345678" { + t.Errorf("GPUUID = %v, want GPU-12345678", event.GPUUID) + } + if event.IsHealthy { + t.Error("Expected IsHealthy to be false") + } + if !event.IsFatal { + t.Error("Expected IsFatal to be true") + } + if len(event.ErrorCodes) != 1 || event.ErrorCodes[0] != "79" { + t.Errorf("ErrorCodes = %v, want [79]", event.ErrorCodes) + } +} + +func TestMockHealthSource(t *testing.T) { + started := false + stopped := false + + source := &mockHealthSource{ + name: "test-source", + startFn: func(ctx context.Context, handler HealthEventHandler) error { + started = true + return nil + }, + stopFn: func() { + stopped = true + }, + } + + if source.Name() != "test-source" { + t.Errorf("Name() = %v, want test-source", source.Name()) + } + + ctx := context.Background() + if err := source.Start(ctx, nil); err != nil { + t.Errorf("Start() error = %v", err) + } + if !started { + t.Error("Expected startFn to be called") + } + + source.Stop() + if !stopped { + t.Error("Expected stopFn to be called") + } +} diff --git a/pkg/version/version.go b/pkg/version/version.go new file mode 100644 index 000000000..4da6eb03f --- /dev/null +++ b/pkg/version/version.go @@ -0,0 +1,80 @@ +// Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package version provides version information for the Device API Server. +// These values are set at build time via ldflags. +package version + +import ( + "fmt" + "runtime" +) + +// Build information set at compile time via -ldflags. +var ( + // Version is the semantic version of the build. + Version = "dev" + + // GitCommit is the git commit SHA at build time. + GitCommit = "unknown" + + // GitTreeState indicates if the git tree was clean or dirty. + GitTreeState = "unknown" + + // BuildDate is the date of the build in ISO 8601 format. + BuildDate = "unknown" +) + +// Info contains version information. +type Info struct { + Version string `json:"version"` + GitCommit string `json:"gitCommit"` + GitTreeState string `json:"gitTreeState"` + BuildDate string `json:"buildDate"` + GoVersion string `json:"goVersion"` + Compiler string `json:"compiler"` + Platform string `json:"platform"` +} + +// Get returns the version information. +func Get() Info { + return Info{ + Version: Version, + GitCommit: GitCommit, + GitTreeState: GitTreeState, + BuildDate: BuildDate, + GoVersion: runtime.Version(), + Compiler: runtime.Compiler, + Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), + } +} + +// String returns version information as a human-readable string. +func (i Info) String() string { + return fmt.Sprintf( + "Version: %s\nGit Commit: %s\nGit Tree State: %s\nBuild Date: %s\nGo Version: %s\nCompiler: %s\nPlatform: %s", + i.Version, + i.GitCommit, + i.GitTreeState, + i.BuildDate, + i.GoVersion, + i.Compiler, + i.Platform, + ) +} + +// Short returns a short version string. +func (i Info) Short() string { + return fmt.Sprintf("%s (%s)", i.Version, i.GitCommit) +} diff --git a/test/integration/test-healthevents.yaml b/test/integration/test-healthevents.yaml new file mode 100644 index 000000000..59f17fd25 --- /dev/null +++ b/test/integration/test-healthevents.yaml @@ -0,0 +1,124 @@ +# Test HealthEvents for Integration Testing +# +# These are sample HealthEvent resources that simulate GPU faults +# for testing the controller chain. +# +# Usage: +# kubectl apply -f test-healthevents.yaml +# kubectl get healthevents -w +# +--- +# Test 1: Fatal XID Error - Should trigger full workflow +# New -> Quarantined -> Drained -> Remediated +apiVersion: nvsentinel.nvidia.com/v1alpha1 +kind: HealthEvent +metadata: + name: test-fatal-xid-79 + labels: + test: integration + scenario: fatal-xid +spec: + source: "nvml-health-monitor" + nodeName: "ip-10-0-0-10" # Target a real worker node + componentClass: "GPU" + checkName: "xid-error-check" + isFatal: true + isHealthy: false + message: "XID 79: GPU has fallen off the bus" + recommendedAction: RESTART_VM + errorCodes: + - "79" + entitiesImpacted: + - type: GPU + value: GPU-TEST-12345678-1234-1234-1234-123456789ABC + detectedAt: "2026-02-04T13:00:00Z" + metadata: + uuid: "GPU-TEST-12345678-1234-1234-1234-123456789ABC" + productName: "NVIDIA A100 80GB" +status: + phase: New +--- +# Test 2: Non-Fatal XID Error - Should resolve immediately +# New -> Resolved (non-fatal, just logged) +apiVersion: nvsentinel.nvidia.com/v1alpha1 +kind: HealthEvent +metadata: + name: test-nonfatal-xid-31 + labels: + test: integration + scenario: nonfatal-xid +spec: + source: "nvml-health-monitor" + nodeName: "ip-10-0-0-236" + componentClass: "GPU" + checkName: "xid-error-check" + isFatal: false + isHealthy: false + message: "XID 31: GPU memory page fault (application error)" + recommendedAction: NONE + errorCodes: + - "31" + entitiesImpacted: + - type: GPU + value: GPU-TEST-87654321-4321-4321-4321-CBA987654321 + detectedAt: "2026-02-04T13:01:00Z" +status: + phase: New +--- +# Test 3: Fatal event with quarantine skip override +# New -> Cancelled (quarantine skipped) +apiVersion: nvsentinel.nvidia.com/v1alpha1 +kind: HealthEvent +metadata: + name: test-fatal-skip-quarantine + labels: + test: integration + scenario: skip-quarantine +spec: + source: "nvml-health-monitor" + nodeName: "ip-10-0-0-81" + componentClass: "GPU" + checkName: "thermal-check" + isFatal: true + isHealthy: false + message: "GPU temperature exceeded threshold" + recommendedAction: CONTACT_SUPPORT + errorCodes: + - "thermal" + entitiesImpacted: + - type: GPU + value: GPU-TEST-THERMAL-1234-5678-9ABC + detectedAt: "2026-02-04T13:02:00Z" + overrides: + quarantine: + skip: true # Skip quarantine, go straight to cancelled +status: + phase: New +--- +# Test 4: CSP Maintenance Event +# Tests the maintenance event flow +apiVersion: nvsentinel.nvidia.com/v1alpha1 +kind: HealthEvent +metadata: + name: test-csp-maintenance + labels: + test: integration + scenario: csp-maintenance +spec: + source: "csp-health-monitor" + nodeName: "ip-10-0-0-236" + componentClass: "GPU" + checkName: "aws-maintenance" + isFatal: true + isHealthy: false + message: "AWS scheduled maintenance notification" + recommendedAction: RESTART_VM + detectedAt: "2026-02-04T13:03:00Z" + maintenance: + csp: aws + eventId: "aws-maint-123456" + maintenanceType: SCHEDULED + scheduledStartTime: "2026-02-05T06:00:00Z" + scheduledEndTime: "2026-02-05T08:00:00Z" +status: + phase: New diff --git a/tests/fault_quarantine_test.go b/tests/fault_quarantine_test.go index fda15d340..d27f9150f 100644 --- a/tests/fault_quarantine_test.go +++ b/tests/fault_quarantine_test.go @@ -23,7 +23,7 @@ import ( "tests/helpers" - "github.com/nvidia/nvsentinel/data-models/pkg/protos" + nvsentinelv1alpha1 "github.com/nvidia/nvsentinel/api/nvsentinel/v1alpha1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" @@ -32,187 +32,212 @@ import ( "sigs.k8s.io/e2e-framework/pkg/features" ) -func TestDontCordonIfEventDoesntMatchCELExpression(t *testing.T) { - feature := features.New("TestCELExpressionFiltering"). - WithLabel("suite", "fault-quarantine-cel") - - var testCtx *helpers.QuarantineTestContext +// TestNonFatalEventDoesNotTriggerQuarantine tests that non-fatal events don't trigger quarantine. +func TestNonFatalEventDoesNotTriggerQuarantine(t *testing.T) { + feature := features.New("TestNonFatalEventDoesNotTriggerQuarantine"). + WithLabel("suite", "quarantine-controller") feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - var newCtx context.Context - newCtx, testCtx = helpers.SetupQuarantineTest(ctx, t, c, "data/managed-by-nvsentinel-configmap.yaml") - return newCtx - }) - - feature.Assess("event doesn't match CEL expression", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client, err := c.NewClient() - require.NoError(t, err) + assert.NoError(t, err) - event := helpers.NewHealthEvent(testCtx.NodeName). - WithCheckName("UnknownCheck"). - WithErrorCode("999") - helpers.SendHealthEvent(ctx, t, event) + nodeName := helpers.SelectTestNodeFromUnusedPool(ctx, t, client) + t.Logf("Selected test node: %s", nodeName) - helpers.AssertQuarantineState(ctx, t, client, testCtx.NodeName, helpers.QuarantineAssertion{ - ExpectCordoned: false, - ExpectAnnotation: false, - }) + // Clean up any existing HealthEvents + helpers.DeleteAllHealthEventCRDs(ctx, t, client) + ctx = context.WithValue(ctx, keyNodeName, nodeName) return ctx }) - feature.Assess("node with ManagedByNVSentinel=false label ignored by CEL", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + feature.Assess("Non-fatal event stays in New phase, node not cordoned", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + nodeName := ctx.Value(keyNodeName).(string) + client, err := c.NewClient() require.NoError(t, err) - err = helpers.SetNodeManagedByNVSentinel(ctx, client, testCtx.NodeName, false) - require.NoError(t, err) + // Create a non-fatal event (isFatal=false) + event := helpers.NewHealthEventCRD(nodeName). + WithSource("e2e-test"). + WithCheckName("GpuWarning"). + WithComponentClass("GPU"). + WithFatal(false). // Non-fatal + WithHealthy(false). + WithErrorCodes("999"). + WithMessage("Non-fatal warning"). + Build() - event := helpers.NewHealthEvent(testCtx.NodeName). - WithErrorCode("79"). - WithMessage("XID error occurred") - helpers.SendHealthEvent(ctx, t, event) + created := helpers.CreateHealthEventCRD(ctx, t, client, event) + t.Logf("Created non-fatal HealthEvent: %s", created.Name) - helpers.AssertNodeNeverQuarantined(ctx, t, client, testCtx.NodeName, true) + ctx = context.WithValue(ctx, keyHealthEventName, created.Name) + + // Assert that the event never reaches Quarantined phase + // QuarantineController should skip non-fatal events + helpers.AssertHealthEventNeverReachesPhase(ctx, t, client, created.Name, nvsentinelv1alpha1.PhaseQuarantined) + + // Verify node was NOT cordoned + node, err := helpers.GetNodeByName(ctx, client, nodeName) + require.NoError(t, err) + assert.False(t, node.Spec.Unschedulable, "node should NOT be cordoned for non-fatal event") return ctx }) feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client, err := c.NewClient() - require.NoError(t, err) + assert.NoError(t, err) - err = helpers.RemoveNodeManagedByNVSentinelLabel(ctx, client, testCtx.NodeName) - require.NoError(t, err) - - return helpers.TeardownQuarantineTest(ctx, t, c) + helpers.DeleteAllHealthEventCRDs(ctx, t, client) + return ctx }) testEnv.Test(t, feature.Feature()) } -func TestPreCordonedNodeHandling(t *testing.T) { - feature := features.New("TestPreCordonedNodeHandling"). - WithLabel("suite", "fault-quarantine-special-modes") - - var testCtx *helpers.QuarantineTestContext +// TestHealthyEventDoesNotTriggerQuarantine tests that healthy events don't trigger quarantine. +func TestHealthyEventDoesNotTriggerQuarantine(t *testing.T) { + feature := features.New("TestHealthyEventDoesNotTriggerQuarantine"). + WithLabel("suite", "quarantine-controller") feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - var newCtx context.Context - newCtx, testCtx = helpers.SetupQuarantineTest(ctx, t, c, "data/basic-matching-configmap.yaml") - client, err := c.NewClient() - require.NoError(t, err) - - t.Logf("Manually cordoning and tainting node %s", testCtx.NodeName) - node, err := helpers.GetNodeByName(ctx, client, testCtx.NodeName) - require.NoError(t, err) + assert.NoError(t, err) - node.Spec.Unschedulable = true - node.Spec.Taints = append(node.Spec.Taints, v1.Taint{ - Key: "manual-taint", - Value: "true", - Effect: v1.TaintEffectNoSchedule, - }) + nodeName := helpers.SelectTestNodeFromUnusedPool(ctx, t, client) + t.Logf("Selected test node: %s", nodeName) - err = client.Resources().Update(ctx, node) - require.NoError(t, err) - t.Logf("Node %s pre-cordoned with manual taint", testCtx.NodeName) + helpers.DeleteAllHealthEventCRDs(ctx, t, client) - return newCtx + ctx = context.WithValue(ctx, keyNodeName, nodeName) + return ctx }) - feature.Assess("FQ adds its taints to pre-cordoned node", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + feature.Assess("Healthy event stays in New phase, node not cordoned", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + nodeName := ctx.Value(keyNodeName).(string) + client, err := c.NewClient() require.NoError(t, err) - event := helpers.NewHealthEvent(testCtx.NodeName). - WithErrorCode("79"). - WithMessage("XID error occurred") - helpers.SendHealthEvent(ctx, t, event) - - t.Log("Waiting for FQ to add its taint to pre-cordoned node") - require.Eventually(t, func() bool { - node, err := helpers.GetNodeByName(ctx, client, testCtx.NodeName) - if err != nil { - t.Logf("failed to get node: %v", err) - return false - } + // Create a healthy event (isHealthy=true) + event := helpers.NewHealthEventCRD(nodeName). + WithSource("e2e-test"). + WithCheckName("GpuXidError"). + WithComponentClass("GPU"). + WithFatal(true). + WithHealthy(true). // Healthy + WithMessage("No errors detected"). + Build() - hasFQTaint := false - hasManualTaint := false + created := helpers.CreateHealthEventCRD(ctx, t, client, event) + t.Logf("Created healthy HealthEvent: %s", created.Name) - for _, taint := range node.Spec.Taints { - if taint.Key == "AggregatedNodeHealth" { - hasFQTaint = true - } - if taint.Key == "manual-taint" { - hasManualTaint = true - } - } + // Assert that the event never reaches Quarantined phase + helpers.AssertHealthEventNeverReachesPhase(ctx, t, client, created.Name, nvsentinelv1alpha1.PhaseQuarantined) - t.Logf("Node state: hasFQTaint=%v, hasManualTaint=%v, cordoned=%v, taints=%+v", - hasFQTaint, hasManualTaint, node.Spec.Unschedulable, node.Spec.Taints) + // Verify node was NOT cordoned + node, err := helpers.GetNodeByName(ctx, client, nodeName) + require.NoError(t, err) + assert.False(t, node.Spec.Unschedulable, "node should NOT be cordoned for healthy event") - if !hasFQTaint { - t.Log("Waiting for FQ taint to be added") - return false - } + return ctx + }) - return hasFQTaint && hasManualTaint && node.Spec.Unschedulable - }, helpers.EventuallyWaitTimeout, helpers.WaitInterval) - t.Log("FQ taint successfully added to pre-cordoned node") + feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client, err := c.NewClient() + assert.NoError(t, err) + helpers.DeleteAllHealthEventCRDs(ctx, t, client) return ctx }) - feature.Assess("FQ annotations added", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + testEnv.Test(t, feature.Feature()) +} + +// TestPreCordonedNodeHandling tests that QuarantineController handles pre-cordoned nodes correctly. +func TestPreCordonedNodeHandling(t *testing.T) { + feature := features.New("TestPreCordonedNodeHandling"). + WithLabel("suite", "quarantine-controller") + + feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client, err := c.NewClient() + assert.NoError(t, err) + + nodeName := helpers.SelectTestNodeFromUnusedPool(ctx, t, client) + t.Logf("Selected test node: %s", nodeName) + + // Pre-cordon the node manually with a taint + t.Logf("Manually cordoning and tainting node %s", nodeName) + node, err := helpers.GetNodeByName(ctx, client, nodeName) require.NoError(t, err) - node, err := helpers.GetNodeByName(ctx, client, testCtx.NodeName) + node.Spec.Unschedulable = true + node.Spec.Taints = append(node.Spec.Taints, v1.Taint{ + Key: "manual-taint", + Value: "true", + Effect: v1.TaintEffectNoSchedule, + }) + + err = client.Resources().Update(ctx, node) require.NoError(t, err) - require.NotNil(t, node.Annotations) + t.Logf("Node %s pre-cordoned with manual taint", nodeName) - _, exists := node.Annotations["quarantineHealthEvent"] - assert.True(t, exists) + helpers.DeleteAllHealthEventCRDs(ctx, t, client) + ctx = context.WithValue(ctx, keyNodeName, nodeName) return ctx }) - feature.Assess("FQ clears its taints on healthy event", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + feature.Assess("QuarantineController processes event on pre-cordoned node", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + nodeName := ctx.Value(keyNodeName).(string) + client, err := c.NewClient() require.NoError(t, err) - helpers.SendHealthyEvent(ctx, t, testCtx.NodeName) + // Create a fatal event + event := helpers.NewHealthEventCRD(nodeName). + WithSource("e2e-test"). + WithCheckName("GpuXidError"). + WithFatal(true). + WithHealthy(false). + WithErrorCodes("79"). + WithMessage("XID error occurred"). + Build() - require.Eventually(t, func() bool { - node, err := helpers.GetNodeByName(ctx, client, testCtx.NodeName) - if err != nil { - return false - } + created := helpers.CreateHealthEventCRD(ctx, t, client, event) + t.Logf("Created HealthEvent: %s", created.Name) - hasFQTaint := false - for _, taint := range node.Spec.Taints { - if taint.Key == "AggregatedNodeHealth" { - hasFQTaint = true - break - } - } + ctx = context.WithValue(ctx, keyHealthEventName, created.Name) - _, hasAnnotation := node.Annotations["quarantineHealthEvent"] + // Wait for QuarantineController to process (should still transition to Quarantined) + helpers.WaitForHealthEventPhase(ctx, t, client, created.Name, nvsentinelv1alpha1.PhaseQuarantined) - return !hasFQTaint && !hasAnnotation - }, helpers.EventuallyWaitTimeout, helpers.WaitInterval) + // Verify the manual taint is preserved + node, err := helpers.GetNodeByName(ctx, client, nodeName) + require.NoError(t, err) + + hasManualTaint := false + for _, taint := range node.Spec.Taints { + if taint.Key == "manual-taint" { + hasManualTaint = true + break + } + } + assert.True(t, hasManualTaint, "manual taint should be preserved") + assert.True(t, node.Spec.Unschedulable, "node should remain cordoned") return ctx }) feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client, err := c.NewClient() - require.NoError(t, err) + assert.NoError(t, err) - node, err := helpers.GetNodeByName(ctx, client, testCtx.NodeName) + nodeName := ctx.Value(keyNodeName).(string) + + // Clean up the node + node, err := helpers.GetNodeByName(ctx, client, nodeName) if err == nil { node.Spec.Unschedulable = false newTaints := []v1.Taint{} @@ -225,184 +250,155 @@ func TestPreCordonedNodeHandling(t *testing.T) { client.Resources().Update(ctx, node) } - return helpers.TeardownQuarantineTest(ctx, t, c) + helpers.DeleteAllHealthEventCRDs(ctx, t, client) + return ctx }) testEnv.Test(t, feature.Feature()) } -func TestCircuitBreakerCursorCreateSkipsAccumulatedEvents(t *testing.T) { - feature := features.New("TestCircuitBreakerCursorCreateSkipsAccumulatedEvents"). - WithLabel("suite", "fault-quarantine-circuit-breaker") - - var testCtx *helpers.QuarantineTestContext +// TestQuarantineSkipOverride tests that quarantine can be skipped via override. +func TestQuarantineSkipOverride(t *testing.T) { + feature := features.New("TestQuarantineSkipOverride"). + WithLabel("suite", "quarantine-controller") feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - // Setup with circuit breaker starting in TRIPPED state - var newCtx context.Context - newCtx, testCtx, _ = helpers.SetupQuarantineTestWithOptions(ctx, t, c, "", &helpers.QuarantineSetupOptions{ - CircuitBreakerState: "TRIPPED", - CircuitBreakerCursorMode: "RESUME", - }) - - t.Logf("Selected test node: %s", testCtx.NodeName) - - // Send event while CB is TRIPPED - this will accumulate in datastore - t.Logf("Sending event for node %s while CB is TRIPPED (should accumulate)", testCtx.NodeName) - event := helpers.NewHealthEvent(testCtx.NodeName). - WithErrorCode("79"). - WithMessage("Accumulated event while CB tripped") - helpers.SendHealthEvent(newCtx, t, event) - - return newCtx - }) - - feature.Assess("reset CB with cursor=CREATE and verify accumulated event skipped", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client, err := c.NewClient() - require.NoError(t, err) + assert.NoError(t, err) - t.Log("Resetting circuit breaker with cursor=CREATE to skip accumulated events") - helpers.SetCircuitBreakerState(ctx, t, c, "CLOSED", "CREATE") + nodeName := helpers.SelectTestNodeFromUnusedPool(ctx, t, client) + t.Logf("Selected test node: %s", nodeName) - t.Logf("Verifying node %s was NOT cordoned (accumulated event should be skipped)", testCtx.NodeName) - helpers.AssertNodeNeverQuarantined(ctx, t, client, testCtx.NodeName, true) + helpers.DeleteAllHealthEventCRDs(ctx, t, client) + ctx = context.WithValue(ctx, keyNodeName, nodeName) return ctx }) - feature.Assess("verify new events ARE processed after cursor=CREATE reset", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + feature.Assess("Event with skip quarantine override doesn't cordon node", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + nodeName := ctx.Value(keyNodeName).(string) + client, err := c.NewClient() require.NoError(t, err) - // Send a NEW event for the same node - this should be processed - t.Logf("Sending NEW event for node %s (should be processed)", testCtx.NodeName) + // Create a fatal event with skip quarantine override + event := helpers.NewHealthEventCRD(nodeName). + WithSource("e2e-test"). + WithCheckName("GpuXidError"). + WithFatal(true). + WithHealthy(false). + WithErrorCodes("79"). + WithMessage("XID error occurred"). + WithSkipQuarantine(true). // Skip quarantine + Build() - event := helpers.NewHealthEvent(testCtx.NodeName). - WithErrorCode("79"). - WithMessage("New event after CB reset") - helpers.SendHealthEvent(ctx, t, event) + created := helpers.CreateHealthEventCRD(ctx, t, client, event) + t.Logf("Created HealthEvent with skip quarantine: %s", created.Name) - // This node SHOULD be cordoned because it's a new event - helpers.AssertQuarantineState(ctx, t, client, testCtx.NodeName, helpers.QuarantineAssertion{ - ExpectCordoned: true, - ExpectAnnotation: true, - }) + // Assert that the event never reaches Quarantined phase + // QuarantineController should respect the skip override + helpers.AssertHealthEventNeverReachesPhase(ctx, t, client, created.Name, nvsentinelv1alpha1.PhaseQuarantined) - t.Logf("Node %s correctly cordoned - new events are being processed", testCtx.NodeName) + // Verify node was NOT cordoned + node, err := helpers.GetNodeByName(ctx, client, nodeName) + require.NoError(t, err) + assert.False(t, node.Spec.Unschedulable, "node should NOT be cordoned when quarantine is skipped") return ctx }) feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - return helpers.TeardownQuarantineTest(ctx, t, c) + client, err := c.NewClient() + assert.NoError(t, err) + + helpers.DeleteAllHealthEventCRDs(ctx, t, client) + return ctx }) testEnv.Test(t, feature.Feature()) } -func TestFaultQuarantineWithProcessingStrategy(t *testing.T) { - feature := features.New("TestFaultQuarantineWithProcessingStrategy"). - WithLabel("suite", "fault-quarantine-with-processing-strategy") - - var testCtx *helpers.QuarantineTestContext +// TestMultipleEventsOnSameNode tests handling of multiple events on the same node. +func TestMultipleEventsOnSameNode(t *testing.T) { + feature := features.New("TestMultipleEventsOnSameNode"). + WithLabel("suite", "quarantine-controller") feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - var newCtx context.Context - newCtx, testCtx = helpers.SetupQuarantineTest(ctx, t, c, "") - return newCtx - }) - - feature.Assess("Check that node is not quarantined for STORE_ONLY events", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client, err := c.NewClient() - require.NoError(t, err) + assert.NoError(t, err) - event := helpers.NewHealthEvent(testCtx.NodeName). - WithErrorCode("79"). - WithMessage("XID error occurred"). - WithAgent(helpers.SYSLOG_HEALTH_MONITOR_AGENT). - WithCheckName("SysLogsXIDError"). - WithProcessingStrategy(int(protos.ProcessingStrategy_STORE_ONLY)) - helpers.SendHealthEvent(ctx, t, event) + nodeName := helpers.SelectTestNodeFromUnusedPool(ctx, t, client) + t.Logf("Selected test node: %s", nodeName) - t.Logf("Node %s should not have condition SysLogsXIDError", testCtx.NodeName) - helpers.EnsureNodeConditionNotPresent(ctx, t, client, testCtx.NodeName, "SysLogsXIDError") + helpers.DeleteAllHealthEventCRDs(ctx, t, client) - helpers.AssertQuarantineState(ctx, t, client, testCtx.NodeName, helpers.QuarantineAssertion{ - ExpectCordoned: false, - ExpectAnnotation: false, - }) + ctx = context.WithValue(ctx, keyNodeName, nodeName) + return ctx + }) - event = helpers.NewHealthEvent(testCtx.NodeName). - WithErrorCode("DCGM_FR_CLOCK_THROTTLE_POWER"). - WithCheckName("GpuPowerWatch"). - WithFatal(false). - WithProcessingStrategy(int(protos.ProcessingStrategy_STORE_ONLY)) - helpers.SendHealthEvent(ctx, t, event) + feature.Assess("Second event on already-quarantined node processes correctly", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + nodeName := ctx.Value(keyNodeName).(string) - t.Logf("Node %s should not have GpuPowerWatch node event", testCtx.NodeName) - helpers.EnsureNodeEventNotPresent(ctx, t, client, testCtx.NodeName, "GpuPowerWatch", "GpuPowerWatchIsNotHealthy") + client, err := c.NewClient() + require.NoError(t, err) - helpers.AssertQuarantineState(ctx, t, client, testCtx.NodeName, helpers.QuarantineAssertion{ - ExpectCordoned: false, - ExpectAnnotation: false, - }) + // Create first fatal event + event1 := helpers.NewHealthEventCRD(nodeName). + WithSource("e2e-test"). + WithCheckName("GpuXidError"). + WithFatal(true). + WithHealthy(false). + WithErrorCodes("79"). + WithMessage("First XID error"). + Build() + + created1 := helpers.CreateHealthEventCRD(ctx, t, client, event1) + t.Logf("Created first HealthEvent: %s", created1.Name) + + // Wait for first event to quarantine the node + helpers.WaitForHealthEventPhase(ctx, t, client, created1.Name, nvsentinelv1alpha1.PhaseQuarantined) + helpers.WaitForNodesCordonState(ctx, t, client, []string{nodeName}, true) + + // Create second fatal event on the same node + event2 := helpers.NewHealthEventCRD(nodeName). + WithSource("e2e-test"). + WithCheckName("GpuMemoryError"). + WithFatal(true). + WithHealthy(false). + WithErrorCodes("31"). + WithMessage("Second memory error"). + Build() + + created2 := helpers.CreateHealthEventCRD(ctx, t, client, event2) + t.Logf("Created second HealthEvent: %s", created2.Name) + + // Second event should also transition to Quarantined (node already cordoned) + helpers.WaitForHealthEventPhase(ctx, t, client, created2.Name, nvsentinelv1alpha1.PhaseQuarantined) + + // Verify node is still cordoned + node, err := helpers.GetNodeByName(ctx, client, nodeName) + require.NoError(t, err) + assert.True(t, node.Spec.Unschedulable, "node should remain cordoned") return ctx }) - feature.Assess("Check that node is quarantined for EXECUTE_REMEDIATION events", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client, err := c.NewClient() - require.NoError(t, err) + assert.NoError(t, err) - event := helpers.NewHealthEvent(testCtx.NodeName). - WithErrorCode("79"). - WithMessage("XID error occurred"). - WithAgent(helpers.SYSLOG_HEALTH_MONITOR_AGENT). - WithCheckName("SysLogsXIDError"). - WithProcessingStrategy(int(protos.ProcessingStrategy_EXECUTE_REMEDIATION)) - helpers.SendHealthEvent(ctx, t, event) - - t.Logf("Node %s should have condition SysLogsXIDError", testCtx.NodeName) - helpers.WaitForNodeConditionWithCheckName(ctx, t, client, testCtx.NodeName, "SysLogsXIDError", "", "SysLogsXIDErrorIsNotHealthy", v1.ConditionTrue) - - helpers.AssertQuarantineState(ctx, t, client, testCtx.NodeName, helpers.QuarantineAssertion{ - ExpectCordoned: true, - ExpectAnnotation: true, - }) + nodeName := ctx.Value(keyNodeName).(string) - event = helpers.NewHealthEvent(testCtx.NodeName). - WithErrorCode("DCGM_FR_CLOCK_THROTTLE_POWER"). - WithCheckName("GpuPowerWatch"). - WithFatal(false). - WithProcessingStrategy(int(protos.ProcessingStrategy_EXECUTE_REMEDIATION)) - helpers.SendHealthEvent(ctx, t, event) - - t.Logf("Node %s should have node event GpuPowerWatch", testCtx.NodeName) - expectedEvent := v1.Event{ - Type: "GpuPowerWatch", - Reason: "GpuPowerWatchIsNotHealthy", - Message: "ErrorCode:DCGM_FR_CLOCK_THROTTLE_POWER GPU:0 Recommended Action=NONE;", + // Uncordon node + node, err := helpers.GetNodeByName(ctx, client, nodeName) + if err == nil && node.Spec.Unschedulable { + node.Spec.Unschedulable = false + client.Resources().Update(ctx, node) } - helpers.WaitForNodeEvent(ctx, t, client, testCtx.NodeName, expectedEvent) - - helpers.AssertQuarantineState(ctx, t, client, testCtx.NodeName, helpers.QuarantineAssertion{ - ExpectCordoned: true, - ExpectAnnotation: true, - }) + helpers.DeleteAllHealthEventCRDs(ctx, t, client) return ctx }) - feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - event := helpers.NewHealthEvent(testCtx.NodeName). - WithErrorCode("79"). - WithHealthy(true). - WithAgent(helpers.SYSLOG_HEALTH_MONITOR_AGENT). - WithCheckName("SysLogsXIDError") - helpers.SendHealthEvent(ctx, t, event) - - return helpers.TeardownQuarantineTest(ctx, t, c) - }) - testEnv.Test(t, feature.Feature()) } diff --git a/tests/fault_remediation_test.go b/tests/fault_remediation_test.go index 9d94667af..1fba00208 100644 --- a/tests/fault_remediation_test.go +++ b/tests/fault_remediation_test.go @@ -20,56 +20,366 @@ package tests import ( "context" "testing" - "time" + nvsentinelv1alpha1 "github.com/nvidia/nvsentinel/api/nvsentinel/v1alpha1" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" "tests/helpers" ) -func TestNewCRsAreCreatedAfterFaultsAreRemediated(t *testing.T) { - feature := features.New("TestNewCRsAreCreatedAfterFaultsAreRemediated"). - WithLabel("suite", "fault-remediation-advanced") +// TestRemediationControllerBasicFlow tests the RemediationController's basic flow. +func TestRemediationControllerBasicFlow(t *testing.T) { + feature := features.New("TestRemediationControllerBasicFlow"). + WithLabel("suite", "remediation-controller") - var testCtx *helpers.RemediationTestContext + feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client, err := c.NewClient() + assert.NoError(t, err) + + nodeName := helpers.SelectTestNodeFromUnusedPool(ctx, t, client) + t.Logf("Selected test node: %s", nodeName) + + // Clean up existing resources + helpers.DeleteAllHealthEventCRDs(ctx, t, client) + helpers.DeleteAllRebootNodeCRs(ctx, t, client) + + ctx = context.WithValue(ctx, keyNodeName, nodeName) + return ctx + }) + + feature.Assess("RemediationController creates RebootNode CR and transitions to Remediated", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + nodeName := ctx.Value(keyNodeName).(string) + + client, err := c.NewClient() + require.NoError(t, err) + + // Create a fatal event that requires remediation + event := helpers.NewHealthEventCRD(nodeName). + WithSource("e2e-test"). + WithCheckName("GpuXidError"). + WithFatal(true). + WithHealthy(false). + WithErrorCodes("79"). + WithMessage("XID error occurred"). + WithRecommendedAction(nvsentinelv1alpha1.ActionRestartVM). + Build() + + created := helpers.CreateHealthEventCRD(ctx, t, client, event) + t.Logf("Created HealthEvent: %s", created.Name) + + ctx = context.WithValue(ctx, keyHealthEventName, created.Name) + + // Wait for event to progress through phases + t.Log("Waiting for Quarantined phase...") + helpers.WaitForHealthEventPhase(ctx, t, client, created.Name, nvsentinelv1alpha1.PhaseQuarantined) + + // DrainController may set Draining/Drained or skip if no pods + t.Log("Waiting for Drained phase...") + helpers.WaitForHealthEventPhase(ctx, t, client, created.Name, nvsentinelv1alpha1.PhaseDrained) + + // Wait for RemediationController to process + t.Log("Waiting for Remediated phase...") + finalEvent := helpers.WaitForHealthEventPhase(ctx, t, client, created.Name, nvsentinelv1alpha1.PhaseRemediated) + + // Verify Remediated condition is set + helpers.AssertRemediatedCondition(t, finalEvent) + + // Verify RebootNode CR was created + rebootNode := helpers.WaitForRebootNodeCR(ctx, t, client, nodeName) + t.Logf("RebootNode CR created and completed: %s", rebootNode.GetName()) + + return ctx + }) + + feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client, err := c.NewClient() + assert.NoError(t, err) + + nodeName := ctx.Value(keyNodeName).(string) + + // Uncordon node + node, err := helpers.GetNodeByName(ctx, client, nodeName) + if err == nil && node.Spec.Unschedulable { + node.Spec.Unschedulable = false + client.Resources().Update(ctx, node) + } + + helpers.DeleteAllHealthEventCRDs(ctx, t, client) + helpers.DeleteAllRebootNodeCRs(ctx, t, client) + + return ctx + }) + + testEnv.Test(t, feature.Feature()) +} + +// TestMultipleRemediationsOnSameNode tests that multiple remediation CRs can be created for the same node. +func TestMultipleRemediationsOnSameNode(t *testing.T) { + feature := features.New("TestMultipleRemediationsOnSameNode"). + WithLabel("suite", "remediation-controller") + + feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client, err := c.NewClient() + assert.NoError(t, err) + + nodeName := helpers.SelectTestNodeFromUnusedPool(ctx, t, client) + t.Logf("Selected test node: %s", nodeName) + + helpers.DeleteAllHealthEventCRDs(ctx, t, client) + helpers.DeleteAllRebootNodeCRs(ctx, t, client) + + ctx = context.WithValue(ctx, keyNodeName, nodeName) + return ctx + }) + + feature.Assess("Second remediation succeeds after first completes", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + nodeName := ctx.Value(keyNodeName).(string) + + client, err := c.NewClient() + require.NoError(t, err) + + // --- First remediation cycle --- + t.Log("=== First remediation cycle ===") + + event1 := helpers.NewHealthEventCRD(nodeName). + WithSource("e2e-test"). + WithCheckName("GpuXidError"). + WithFatal(true). + WithHealthy(false). + WithErrorCodes("79"). + WithRecommendedAction(nvsentinelv1alpha1.ActionRestartVM). + Build() + + created1 := helpers.CreateHealthEventCRD(ctx, t, client, event1) + t.Logf("Created first HealthEvent: %s", created1.Name) + + // Wait for remediation to complete + helpers.WaitForHealthEventPhase(ctx, t, client, created1.Name, nvsentinelv1alpha1.PhaseRemediated) + + cr1 := helpers.WaitForRebootNodeCR(ctx, t, client, nodeName) + t.Logf("First RebootNode CR completed: %s", cr1.GetName()) + + // Send healthy event to resolve + helpers.SendHealthyEventViaCRD(ctx, t, client, nodeName) + + // Wait for first event to be resolved + helpers.WaitForHealthEventPhase(ctx, t, client, created1.Name, nvsentinelv1alpha1.PhaseResolved) + + // Uncordon node for next cycle + node, err := helpers.GetNodeByName(ctx, client, nodeName) + require.NoError(t, err) + if node.Spec.Unschedulable { + node.Spec.Unschedulable = false + err = client.Resources().Update(ctx, node) + require.NoError(t, err) + } + + // --- Second remediation cycle --- + t.Log("=== Second remediation cycle ===") + + event2 := helpers.NewHealthEventCRD(nodeName). + WithSource("e2e-test"). + WithCheckName("GpuMemoryError"). + WithFatal(true). + WithHealthy(false). + WithErrorCodes("31"). + WithRecommendedAction(nvsentinelv1alpha1.ActionRestartVM). + Build() + + created2 := helpers.CreateHealthEventCRD(ctx, t, client, event2) + t.Logf("Created second HealthEvent: %s", created2.Name) + + // Wait for second remediation + helpers.WaitForHealthEventPhase(ctx, t, client, created2.Name, nvsentinelv1alpha1.PhaseRemediated) + + // Verify we now have 2 completed RebootNode CRs + crList, err := helpers.GetRebootNodeCRsForNode(ctx, client, nodeName) + require.NoError(t, err) + assert.Len(t, crList, 2, "should have 2 completed RebootNode CRs") + + t.Logf("Successfully created %d RebootNode CRs for node %s", len(crList), nodeName) + + return ctx + }) + + feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client, err := c.NewClient() + assert.NoError(t, err) + + nodeName := ctx.Value(keyNodeName).(string) + + // Uncordon node + node, err := helpers.GetNodeByName(ctx, client, nodeName) + if err == nil && node.Spec.Unschedulable { + node.Spec.Unschedulable = false + client.Resources().Update(ctx, node) + } + + helpers.DeleteAllHealthEventCRDs(ctx, t, client) + helpers.DeleteAllRebootNodeCRs(ctx, t, client) + + return ctx + }) + + testEnv.Test(t, feature.Feature()) +} + +// TestContactSupportDoesNotTriggerRemediation tests that CONTACT_SUPPORT events skip remediation. +func TestContactSupportDoesNotTriggerRemediation(t *testing.T) { + feature := features.New("TestContactSupportDoesNotTriggerRemediation"). + WithLabel("suite", "remediation-controller") + + feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client, err := c.NewClient() + assert.NoError(t, err) + + nodeName := helpers.SelectTestNodeFromUnusedPool(ctx, t, client) + t.Logf("Selected test node: %s", nodeName) + + helpers.DeleteAllHealthEventCRDs(ctx, t, client) + helpers.DeleteAllRebootNodeCRs(ctx, t, client) + + ctx = context.WithValue(ctx, keyNodeName, nodeName) + return ctx + }) + + feature.Assess("CONTACT_SUPPORT event does not create RebootNode CR", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + nodeName := ctx.Value(keyNodeName).(string) + + client, err := c.NewClient() + require.NoError(t, err) + + // Create an event with CONTACT_SUPPORT action (no automatic remediation) + event := helpers.NewHealthEventCRD(nodeName). + WithSource("e2e-test"). + WithCheckName("GpuXidError"). + WithFatal(true). + WithHealthy(false). + WithErrorCodes("145"). // Unsupported XID + WithRecommendedAction(nvsentinelv1alpha1.ActionContactSupport). + Build() + + created := helpers.CreateHealthEventCRD(ctx, t, client, event) + t.Logf("Created HealthEvent with CONTACT_SUPPORT: %s", created.Name) + + // Wait for quarantine and drain + helpers.WaitForHealthEventPhase(ctx, t, client, created.Name, nvsentinelv1alpha1.PhaseQuarantined) + helpers.WaitForHealthEventPhase(ctx, t, client, created.Name, nvsentinelv1alpha1.PhaseDrained) + + // Verify NO RebootNode CR is created (CONTACT_SUPPORT = manual intervention required) + helpers.WaitForNoRebootNodeCR(ctx, t, client, nodeName) + t.Log("Verified no RebootNode CR created for CONTACT_SUPPORT event") + + // Event should NOT reach Remediated phase + helpers.AssertHealthEventNeverReachesPhase(ctx, t, client, created.Name, nvsentinelv1alpha1.PhaseRemediated) + + return ctx + }) + + feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client, err := c.NewClient() + assert.NoError(t, err) + + nodeName := ctx.Value(keyNodeName).(string) + + // Uncordon node + node, err := helpers.GetNodeByName(ctx, client, nodeName) + if err == nil && node.Spec.Unschedulable { + node.Spec.Unschedulable = false + client.Resources().Update(ctx, node) + } + + helpers.DeleteAllHealthEventCRDs(ctx, t, client) + helpers.DeleteAllRebootNodeCRs(ctx, t, client) + + return ctx + }) + + testEnv.Test(t, feature.Feature()) +} + +// TestFullPhaseSequenceToResolved tests the complete lifecycle from New to Resolved. +func TestFullPhaseSequenceToResolved(t *testing.T) { + feature := features.New("TestFullPhaseSequenceToResolved"). + WithLabel("suite", "remediation-controller") feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - var newCtx context.Context - newCtx, testCtx = helpers.SetupFaultRemediationTest(ctx, t, c, "") - return newCtx + client, err := c.NewClient() + assert.NoError(t, err) + + nodeName := helpers.SelectTestNodeFromUnusedPool(ctx, t, client) + t.Logf("Selected test node: %s", nodeName) + + helpers.DeleteAllHealthEventCRDs(ctx, t, client) + helpers.DeleteAllRebootNodeCRs(ctx, t, client) + + ctx = context.WithValue(ctx, keyNodeName, nodeName) + return ctx }) - feature.Assess("new CR should be created on the same node after success", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + feature.Assess("HealthEvent progresses through full lifecycle", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + nodeName := ctx.Value(keyNodeName).(string) + client, err := c.NewClient() require.NoError(t, err) - // Trigger first fault and wait for CR - helpers.TriggerFullRemediationFlow(ctx, t, client, testCtx.NodeName, 15) - cr1 := helpers.WaitForRebootNodeCR(ctx, t, client, testCtx.NodeName) - t.Logf("First CR created and completed: %s", cr1.GetName()) + // Create fatal event + event := helpers.NewHealthEventCRD(nodeName). + WithSource("e2e-test"). + WithCheckName("GpuXidError"). + WithFatal(true). + WithHealthy(false). + WithErrorCodes("79"). + WithRecommendedAction(nvsentinelv1alpha1.ActionRestartVM). + Build() + + created := helpers.CreateHealthEventCRD(ctx, t, client, event) + t.Logf("Created HealthEvent: %s", created.Name) + + // Define full expected phase sequence + sequence := helpers.ExpectedPhaseSequence{ + nvsentinelv1alpha1.PhaseQuarantined, + nvsentinelv1alpha1.PhaseDrained, + nvsentinelv1alpha1.PhaseRemediated, + } - // Send healthy event to clear the fault - helpers.SendHealthyEvent(ctx, t, testCtx.NodeName) - time.Sleep(10 * time.Second) + // Wait for sequence up to Remediated + helpers.WaitForHealthEventPhaseSequence(ctx, t, client, created.Name, sequence) + t.Log("Reached Remediated phase") - // Trigger second fault - helpers.TriggerFullRemediationFlow(ctx, t, client, testCtx.NodeName, 15) + // Send healthy event to trigger resolution + helpers.SendHealthyEventViaCRD(ctx, t, client, nodeName) - // Verify that 2 CRs were created - require.Eventually(t, func() bool { - crList, err := helpers.GetRebootNodeCRsForNode(ctx, client, testCtx.NodeName) - if err != nil { - return false - } - return len(crList) == 2 - }, helpers.EventuallyWaitTimeout, helpers.WaitInterval, "should have 2 completed CRs") + // Wait for Resolved phase + finalEvent := helpers.WaitForHealthEventPhase(ctx, t, client, created.Name, nvsentinelv1alpha1.PhaseResolved) + + // Verify ResolvedAt timestamp is set + helpers.AssertResolvedAtSet(t, finalEvent) + + t.Log("Successfully verified full phase sequence: New → Quarantined → Drained → Remediated → Resolved") return ctx }) feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - return helpers.TeardownFaultRemediation(ctx, t, c) + client, err := c.NewClient() + assert.NoError(t, err) + + nodeName := ctx.Value(keyNodeName).(string) + + // Uncordon node + node, err := helpers.GetNodeByName(ctx, client, nodeName) + if err == nil && node.Spec.Unschedulable { + node.Spec.Unschedulable = false + client.Resources().Update(ctx, node) + } + + helpers.DeleteAllHealthEventCRDs(ctx, t, client) + helpers.DeleteAllRebootNodeCRs(ctx, t, client) + + return ctx }) testEnv.Test(t, feature.Feature()) diff --git a/tests/go.mod b/tests/go.mod index 8fd1e895a..796562c1b 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -8,6 +8,7 @@ require ( github.com/BurntSushi/toml v1.6.0 github.com/go-logr/logr v1.4.3 github.com/hashicorp/go-retryablehttp v0.7.8 + github.com/nvidia/nvsentinel/api v0.0.0 github.com/nvidia/nvsentinel/commons v0.0.0 github.com/nvidia/nvsentinel/data-models v0.0.0 github.com/prometheus/client_golang v1.23.2 @@ -90,6 +91,8 @@ require ( ) // Local replacements for internal modules +replace github.com/nvidia/nvsentinel/api => ../api + replace github.com/nvidia/nvsentinel/data-models => ../data-models replace github.com/nvidia/nvsentinel/store-client => ../store-client diff --git a/tests/helpers/healthevent_crd.go b/tests/helpers/healthevent_crd.go new file mode 100644 index 000000000..efa310c70 --- /dev/null +++ b/tests/helpers/healthevent_crd.go @@ -0,0 +1,605 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package helpers + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/e2e-framework/klient" + + nvsentinelv1alpha1 "github.com/nvidia/nvsentinel/api/nvsentinel/v1alpha1" +) + +// HealthEventCRDBuilder provides a fluent interface for building HealthEvent CRDs. +type HealthEventCRDBuilder struct { + event *nvsentinelv1alpha1.HealthEvent +} + +// NewHealthEventCRD creates a new HealthEvent CRD builder with sensible defaults. +func NewHealthEventCRD(nodeName string) *HealthEventCRDBuilder { + return &HealthEventCRDBuilder{ + event: &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("test-%s-", nodeName), + }, + Spec: nvsentinelv1alpha1.HealthEventSpec{ + Source: "e2e-test", + NodeName: nodeName, + ComponentClass: "GPU", + CheckName: "GpuXidError", + IsFatal: true, + IsHealthy: false, + DetectedAt: metav1.Now(), + }, + }, + } +} + +// WithName sets the exact name (instead of GenerateName). +func (b *HealthEventCRDBuilder) WithName(name string) *HealthEventCRDBuilder { + b.event.ObjectMeta.Name = name + b.event.ObjectMeta.GenerateName = "" + return b +} + +// WithSource sets the source of the health event. +func (b *HealthEventCRDBuilder) WithSource(source string) *HealthEventCRDBuilder { + b.event.Spec.Source = source + return b +} + +// WithCheckName sets the check name. +func (b *HealthEventCRDBuilder) WithCheckName(checkName string) *HealthEventCRDBuilder { + b.event.Spec.CheckName = checkName + return b +} + +// WithComponentClass sets the component class. +func (b *HealthEventCRDBuilder) WithComponentClass(class string) *HealthEventCRDBuilder { + b.event.Spec.ComponentClass = class + return b +} + +// WithFatal sets the isFatal flag. +func (b *HealthEventCRDBuilder) WithFatal(isFatal bool) *HealthEventCRDBuilder { + b.event.Spec.IsFatal = isFatal + return b +} + +// WithHealthy sets the isHealthy flag. +func (b *HealthEventCRDBuilder) WithHealthy(isHealthy bool) *HealthEventCRDBuilder { + b.event.Spec.IsHealthy = isHealthy + return b +} + +// WithMessage sets the message. +func (b *HealthEventCRDBuilder) WithMessage(message string) *HealthEventCRDBuilder { + b.event.Spec.Message = message + return b +} + +// WithRecommendedAction sets the recommended action. +func (b *HealthEventCRDBuilder) WithRecommendedAction(action nvsentinelv1alpha1.RecommendedAction) *HealthEventCRDBuilder { + b.event.Spec.RecommendedAction = action + return b +} + +// WithErrorCodes sets the error codes. +func (b *HealthEventCRDBuilder) WithErrorCodes(codes ...string) *HealthEventCRDBuilder { + b.event.Spec.ErrorCodes = codes + return b +} + +// WithEntity adds an entity to the entities impacted list. +func (b *HealthEventCRDBuilder) WithEntity(entityType, entityValue string) *HealthEventCRDBuilder { + b.event.Spec.EntitiesImpacted = append(b.event.Spec.EntitiesImpacted, nvsentinelv1alpha1.Entity{ + Type: entityType, + Value: entityValue, + }) + return b +} + +// WithMetadata adds a metadata key-value pair. +func (b *HealthEventCRDBuilder) WithMetadata(key, value string) *HealthEventCRDBuilder { + if b.event.Spec.Metadata == nil { + b.event.Spec.Metadata = make(map[string]string) + } + b.event.Spec.Metadata[key] = value + return b +} + +// WithSkipQuarantine sets the quarantine skip override. +func (b *HealthEventCRDBuilder) WithSkipQuarantine(skip bool) *HealthEventCRDBuilder { + if b.event.Spec.Overrides == nil { + b.event.Spec.Overrides = &nvsentinelv1alpha1.BehaviourOverrides{} + } + if b.event.Spec.Overrides.Quarantine == nil { + b.event.Spec.Overrides.Quarantine = &nvsentinelv1alpha1.ActionOverride{} + } + b.event.Spec.Overrides.Quarantine.Skip = skip + return b +} + +// WithSkipDrain sets the drain skip override. +func (b *HealthEventCRDBuilder) WithSkipDrain(skip bool) *HealthEventCRDBuilder { + if b.event.Spec.Overrides == nil { + b.event.Spec.Overrides = &nvsentinelv1alpha1.BehaviourOverrides{} + } + if b.event.Spec.Overrides.Drain == nil { + b.event.Spec.Overrides.Drain = &nvsentinelv1alpha1.ActionOverride{} + } + b.event.Spec.Overrides.Drain.Skip = skip + return b +} + +// Build returns the constructed HealthEvent. +func (b *HealthEventCRDBuilder) Build() *nvsentinelv1alpha1.HealthEvent { + return b.event +} + +// ============================================================================= +// CRD Operations +// ============================================================================= + +// CreateHealthEventCRD creates a HealthEvent CRD in the cluster. +func CreateHealthEventCRD( + ctx context.Context, t *testing.T, c klient.Client, event *nvsentinelv1alpha1.HealthEvent, +) *nvsentinelv1alpha1.HealthEvent { + t.Helper() + t.Logf("Creating HealthEvent CRD for node %s: checkName=%s, isFatal=%v", + event.Spec.NodeName, event.Spec.CheckName, event.Spec.IsFatal) + + err := c.Resources().Create(ctx, event) + require.NoError(t, err, "failed to create HealthEvent CRD") + + t.Logf("Created HealthEvent CRD: %s", event.Name) + return event +} + +// GetHealthEventCRD retrieves a HealthEvent CRD by name. +func GetHealthEventCRD( + ctx context.Context, c klient.Client, name string, +) (*nvsentinelv1alpha1.HealthEvent, error) { + event := &nvsentinelv1alpha1.HealthEvent{} + err := c.Resources().Get(ctx, name, "", event) + if err != nil { + return nil, fmt.Errorf("failed to get HealthEvent %s: %w", name, err) + } + return event, nil +} + +// DeleteHealthEventCRD deletes a HealthEvent CRD by name. +func DeleteHealthEventCRD(ctx context.Context, t *testing.T, c klient.Client, name string) { + t.Helper() + t.Logf("Deleting HealthEvent CRD: %s", name) + + event := &nvsentinelv1alpha1.HealthEvent{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + + err := c.Resources().Delete(ctx, event) + if err != nil && !apierrors.IsNotFound(err) { + t.Logf("Warning: failed to delete HealthEvent %s: %v", name, err) + } +} + +// ListHealthEventCRDs lists all HealthEvent CRDs in the cluster. +func ListHealthEventCRDs( + ctx context.Context, c klient.Client, +) (*nvsentinelv1alpha1.HealthEventList, error) { + list := &nvsentinelv1alpha1.HealthEventList{} + err := c.Resources().List(ctx, list) + if err != nil { + return nil, fmt.Errorf("failed to list HealthEvents: %w", err) + } + return list, nil +} + +// ListHealthEventCRDsForNode lists all HealthEvent CRDs for a specific node. +func ListHealthEventCRDsForNode( + ctx context.Context, c klient.Client, nodeName string, +) ([]nvsentinelv1alpha1.HealthEvent, error) { + list, err := ListHealthEventCRDs(ctx, c) + if err != nil { + return nil, err + } + + var result []nvsentinelv1alpha1.HealthEvent + for _, event := range list.Items { + if event.Spec.NodeName == nodeName { + result = append(result, event) + } + } + return result, nil +} + +// DeleteAllHealthEventCRDs deletes all HealthEvent CRDs in the cluster. +func DeleteAllHealthEventCRDs(ctx context.Context, t *testing.T, c klient.Client) { + t.Helper() + t.Log("Deleting all HealthEvent CRDs") + + list, err := ListHealthEventCRDs(ctx, c) + if err != nil { + t.Logf("Warning: failed to list HealthEvents: %v", err) + return + } + + for _, event := range list.Items { + DeleteHealthEventCRD(ctx, t, c, event.Name) + } + + t.Logf("Deleted %d HealthEvent CRD(s)", len(list.Items)) +} + +// ============================================================================= +// Phase Waiting +// ============================================================================= + +// WaitForHealthEventPhase waits for a HealthEvent to reach the specified phase. +func WaitForHealthEventPhase( + ctx context.Context, t *testing.T, c klient.Client, name string, expectedPhase nvsentinelv1alpha1.HealthEventPhase, +) *nvsentinelv1alpha1.HealthEvent { + t.Helper() + t.Logf("Waiting for HealthEvent %s to reach phase %s", name, expectedPhase) + + var result *nvsentinelv1alpha1.HealthEvent + + require.Eventually(t, func() bool { + event, err := GetHealthEventCRD(ctx, c, name) + if err != nil { + t.Logf("Failed to get HealthEvent %s: %v", name, err) + return false + } + + currentPhase := event.Status.Phase + // Handle empty phase as "New" (status not yet set) + if currentPhase == "" { + currentPhase = nvsentinelv1alpha1.PhaseNew + } + + t.Logf("HealthEvent %s: current phase=%s, expected=%s", name, currentPhase, expectedPhase) + + if currentPhase == expectedPhase { + result = event + return true + } + return false + }, EventuallyWaitTimeout, WaitInterval, + "HealthEvent %s should reach phase %s", name, expectedPhase) + + return result +} + +// WaitForHealthEventPhaseNotEqual waits for a HealthEvent to NOT be in the specified phase. +func WaitForHealthEventPhaseNotEqual( + ctx context.Context, t *testing.T, c klient.Client, name string, notExpectedPhase nvsentinelv1alpha1.HealthEventPhase, +) *nvsentinelv1alpha1.HealthEvent { + t.Helper() + t.Logf("Waiting for HealthEvent %s to leave phase %s", name, notExpectedPhase) + + var result *nvsentinelv1alpha1.HealthEvent + + require.Eventually(t, func() bool { + event, err := GetHealthEventCRD(ctx, c, name) + if err != nil { + t.Logf("Failed to get HealthEvent %s: %v", name, err) + return false + } + + currentPhase := event.Status.Phase + if currentPhase == "" { + currentPhase = nvsentinelv1alpha1.PhaseNew + } + + if currentPhase != notExpectedPhase { + t.Logf("HealthEvent %s: phase changed from %s to %s", name, notExpectedPhase, currentPhase) + result = event + return true + } + return false + }, EventuallyWaitTimeout, WaitInterval, + "HealthEvent %s should leave phase %s", name, notExpectedPhase) + + return result +} + +// WaitForHealthEventCondition waits for a HealthEvent to have a specific condition. +func WaitForHealthEventCondition( + ctx context.Context, t *testing.T, c klient.Client, name string, + conditionType nvsentinelv1alpha1.HealthEventConditionType, expectedStatus metav1.ConditionStatus, +) *nvsentinelv1alpha1.HealthEvent { + t.Helper() + t.Logf("Waiting for HealthEvent %s to have condition %s=%s", name, conditionType, expectedStatus) + + var result *nvsentinelv1alpha1.HealthEvent + + require.Eventually(t, func() bool { + event, err := GetHealthEventCRD(ctx, c, name) + if err != nil { + t.Logf("Failed to get HealthEvent %s: %v", name, err) + return false + } + + for _, condition := range event.Status.Conditions { + if condition.Type == conditionType && condition.Status == expectedStatus { + t.Logf("HealthEvent %s has condition %s=%s", name, conditionType, expectedStatus) + result = event + return true + } + } + + t.Logf("HealthEvent %s: condition %s not yet %s", name, conditionType, expectedStatus) + return false + }, EventuallyWaitTimeout, WaitInterval, + "HealthEvent %s should have condition %s=%s", name, conditionType, expectedStatus) + + return result +} + +// ============================================================================= +// Assertions +// ============================================================================= + +// AssertHealthEventPhase asserts that a HealthEvent is in the expected phase. +func AssertHealthEventPhase( + t *testing.T, event *nvsentinelv1alpha1.HealthEvent, expectedPhase nvsentinelv1alpha1.HealthEventPhase, +) { + t.Helper() + + actualPhase := event.Status.Phase + if actualPhase == "" { + actualPhase = nvsentinelv1alpha1.PhaseNew + } + + require.Equal(t, expectedPhase, actualPhase, + "HealthEvent %s should be in phase %s, but is in %s", event.Name, expectedPhase, actualPhase) +} + +// AssertHealthEventNotExists asserts that a HealthEvent does not exist. +func AssertHealthEventNotExists(ctx context.Context, t *testing.T, c klient.Client, name string) { + t.Helper() + + _, err := GetHealthEventCRD(ctx, c, name) + require.True(t, apierrors.IsNotFound(err), + "HealthEvent %s should not exist, but it does", name) +} + +// AssertHealthEventHasCondition asserts that a HealthEvent has a specific condition. +func AssertHealthEventHasCondition( + t *testing.T, event *nvsentinelv1alpha1.HealthEvent, + conditionType nvsentinelv1alpha1.HealthEventConditionType, expectedStatus metav1.ConditionStatus, +) { + t.Helper() + + for _, condition := range event.Status.Conditions { + if condition.Type == conditionType { + require.Equal(t, expectedStatus, condition.Status, + "HealthEvent %s condition %s should be %s, but is %s", + event.Name, conditionType, expectedStatus, condition.Status) + return + } + } + + t.Fatalf("HealthEvent %s does not have condition %s", event.Name, conditionType) +} + +// AssertHealthEventNeverReachesPhase asserts that a HealthEvent never reaches a specific phase. +func AssertHealthEventNeverReachesPhase( + ctx context.Context, t *testing.T, c klient.Client, name string, phase nvsentinelv1alpha1.HealthEventPhase, +) { + t.Helper() + t.Logf("Asserting HealthEvent %s never reaches phase %s", name, phase) + + require.Never(t, func() bool { + event, err := GetHealthEventCRD(ctx, c, name) + if err != nil { + return false + } + + currentPhase := event.Status.Phase + if currentPhase == "" { + currentPhase = nvsentinelv1alpha1.PhaseNew + } + + if currentPhase == phase { + t.Logf("ERROR: HealthEvent %s reached phase %s (should not happen)", name, phase) + return true + } + return false + }, NeverWaitTimeout, WaitInterval, + "HealthEvent %s should never reach phase %s", name, phase) +} + +// ============================================================================= +// Phase Sequence Tracking +// ============================================================================= + +// ExpectedPhaseSequence defines an expected sequence of phase transitions. +type ExpectedPhaseSequence []nvsentinelv1alpha1.HealthEventPhase + +// WaitForHealthEventPhaseSequence waits for a HealthEvent to progress through a sequence of phases. +// Returns the final HealthEvent once all phases have been observed. +func WaitForHealthEventPhaseSequence( + ctx context.Context, t *testing.T, c klient.Client, name string, sequence ExpectedPhaseSequence, +) *nvsentinelv1alpha1.HealthEvent { + t.Helper() + t.Logf("Waiting for HealthEvent %s to progress through phases: %v", name, sequence) + + var result *nvsentinelv1alpha1.HealthEvent + + for i, expectedPhase := range sequence { + t.Logf("Waiting for phase %d/%d: %s", i+1, len(sequence), expectedPhase) + result = WaitForHealthEventPhase(ctx, t, c, name, expectedPhase) + t.Logf("✓ Phase %d/%d reached: %s", i+1, len(sequence), expectedPhase) + } + + return result +} + +// ============================================================================= +// Cleanup Helpers +// ============================================================================= + +// CleanupHealthEventAndNode cleans up a HealthEvent and uncordons the associated node. +func CleanupHealthEventAndNode(ctx context.Context, t *testing.T, c klient.Client, eventName, nodeName string) { + t.Helper() + t.Logf("Cleaning up HealthEvent %s and node %s", eventName, nodeName) + + // Delete the HealthEvent + DeleteHealthEventCRD(ctx, t, c, eventName) + + // Uncordon the node + node, err := GetNodeByName(ctx, c, nodeName) + if err != nil { + t.Logf("Warning: failed to get node %s: %v", nodeName, err) + return + } + + if node.Spec.Unschedulable { + node.Spec.Unschedulable = false + err = c.Resources().Update(ctx, node) + if err != nil { + t.Logf("Warning: failed to uncordon node %s: %v", nodeName, err) + } else { + t.Logf("Uncordoned node %s", nodeName) + } + } +} + +// ============================================================================= +// Condition Assertions +// ============================================================================= + +// AssertNodeQuarantinedCondition asserts that the NodeQuarantined condition is set. +func AssertNodeQuarantinedCondition(t *testing.T, event *nvsentinelv1alpha1.HealthEvent) { + t.Helper() + AssertHealthEventHasCondition(t, event, nvsentinelv1alpha1.ConditionNodeQuarantined, metav1.ConditionTrue) +} + +// AssertPodsDrainedCondition asserts that the PodsDrained condition is set. +func AssertPodsDrainedCondition(t *testing.T, event *nvsentinelv1alpha1.HealthEvent) { + t.Helper() + AssertHealthEventHasCondition(t, event, nvsentinelv1alpha1.ConditionPodsDrained, metav1.ConditionTrue) +} + +// AssertRemediatedCondition asserts that the Remediated condition is set. +func AssertRemediatedCondition(t *testing.T, event *nvsentinelv1alpha1.HealthEvent) { + t.Helper() + AssertHealthEventHasCondition(t, event, nvsentinelv1alpha1.ConditionRemediated, metav1.ConditionTrue) +} + +// AssertResolvedAtSet asserts that the resolvedAt timestamp is set. +func AssertResolvedAtSet(t *testing.T, event *nvsentinelv1alpha1.HealthEvent) { + t.Helper() + require.NotNil(t, event.Status.ResolvedAt, + "HealthEvent %s should have resolvedAt set", event.Name) + require.False(t, event.Status.ResolvedAt.IsZero(), + "HealthEvent %s should have non-zero resolvedAt", event.Name) +} + +// GetConditionLastTransitionTime returns the last transition time for a condition. +func GetConditionLastTransitionTime( + event *nvsentinelv1alpha1.HealthEvent, conditionType nvsentinelv1alpha1.HealthEventConditionType, +) *metav1.Time { + for _, condition := range event.Status.Conditions { + if condition.Type == conditionType { + return &condition.LastTransitionTime + } + } + return nil +} + +// ============================================================================= +// Integration with Old Test Patterns +// ============================================================================= + +// SendHealthEventViaCRD creates a HealthEvent CRD equivalent to the old SendHealthEvent helper. +// This provides a migration path from HTTP-based event sending to CRD-based. +func SendHealthEventViaCRD( + ctx context.Context, t *testing.T, c klient.Client, template *HealthEventTemplate, +) *nvsentinelv1alpha1.HealthEvent { + t.Helper() + + builder := NewHealthEventCRD(template.NodeName). + WithSource(template.Agent). + WithCheckName(template.CheckName). + WithComponentClass(template.ComponentClass). + WithFatal(template.IsFatal). + WithHealthy(template.IsHealthy). + WithMessage(template.Message) + + // Add error codes + if len(template.ErrorCode) > 0 { + builder.WithErrorCodes(template.ErrorCode...) + } + + // Add entities + for _, entity := range template.EntitiesImpacted { + builder.WithEntity(entity.EntityType, entity.EntityValue) + } + + // Add metadata + for k, v := range template.Metadata { + builder.WithMetadata(k, v) + } + + event := builder.Build() + return CreateHealthEventCRD(ctx, t, c, event) +} + +// SendHealthyEventViaCRD creates a healthy HealthEvent CRD equivalent to the old SendHealthyEvent helper. +func SendHealthyEventViaCRD(ctx context.Context, t *testing.T, c klient.Client, nodeName string) *nvsentinelv1alpha1.HealthEvent { + t.Helper() + t.Logf("Creating healthy HealthEvent CRD for node %s", nodeName) + + event := NewHealthEventCRD(nodeName). + WithHealthy(true). + WithFatal(false). + WithMessage("No health failures"). + Build() + + return CreateHealthEventCRD(ctx, t, c, event) +} + +// TriggerFullRemediationFlowViaCRD creates a fatal HealthEvent and waits for quarantine. +// This is the CRD equivalent of TriggerFullRemediationFlow. +func TriggerFullRemediationFlowViaCRD( + ctx context.Context, t *testing.T, c klient.Client, nodeName string, +) *nvsentinelv1alpha1.HealthEvent { + t.Helper() + t.Logf("Triggering full remediation flow via CRD for node %s", nodeName) + + event := NewHealthEventCRD(nodeName). + WithFatal(true). + WithCheckName("GpuXidError"). + WithErrorCodes("79"). + WithRecommendedAction(nvsentinelv1alpha1.ActionRestartVM). + Build() + + created := CreateHealthEventCRD(ctx, t, c, event) + + // Wait for quarantine + WaitForHealthEventPhase(ctx, t, c, created.Name, nvsentinelv1alpha1.PhaseQuarantined) + WaitForNodesCordonState(ctx, t, c, []string{nodeName}, true) + + return created +} diff --git a/tests/main_test.go b/tests/main_test.go index ba4c39da9..b781da401 100644 --- a/tests/main_test.go +++ b/tests/main_test.go @@ -26,6 +26,8 @@ import ( "sigs.k8s.io/e2e-framework/pkg/env" "sigs.k8s.io/e2e-framework/pkg/envconf" kwokv1alpha1 "sigs.k8s.io/kwok/pkg/apis/v1alpha1" + + nvsentinelv1alpha1 "github.com/nvidia/nvsentinel/api/nvsentinel/v1alpha1" ) var testEnv env.Environment @@ -36,9 +38,10 @@ var testEnv env.Environment type testContextKey int const ( - keyNodeName testContextKey = iota //lint:ignore U1000 Used by test files with build tags - keyNamespace //lint:ignore U1000 Used by test files with build tags - keyPodName //lint:ignore U1000 Used by test files with build tags + keyNodeName testContextKey = iota //lint:ignore U1000 Used by test files with build tags + keyNamespace //lint:ignore U1000 Used by test files with build tags + keyPodName //lint:ignore U1000 Used by test files with build tags + keyHealthEventName //lint:ignore U1000 Used by test files with build tags ) // TestMain sets up the test environment and makes testEnv available @@ -55,6 +58,11 @@ func TestMain(m *testing.M) { panic(fmt.Sprintf("failed to register KWOK types: %v", err)) } + // Register NVSentinel HealthEvent CRD types for CRD-based E2E tests + if err := nvsentinelv1alpha1.AddToScheme(cfg.Client().Resources().GetScheme()); err != nil { + panic(fmt.Sprintf("failed to register NVSentinel types: %v", err)) + } + testEnv = env.NewWithConfig(cfg) os.Exit(testEnv.Run(m)) } diff --git a/tests/node_drainer_test.go b/tests/node_drainer_test.go index 7d8f84088..8a836d8bd 100644 --- a/tests/node_drainer_test.go +++ b/tests/node_drainer_test.go @@ -20,324 +20,396 @@ package tests import ( "context" "testing" - "time" "tests/helpers" + nvsentinelv1alpha1 "github.com/nvidia/nvsentinel/api/nvsentinel/v1alpha1" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" - - "github.com/nvidia/nvsentinel/commons/pkg/statemanager" ) -func TestNodeDrainerEvictionModes(t *testing.T) { - feature := features.New("TestNodeDrainerEvictionModes"). - WithLabel("suite", "node-drainer") - - var testCtx *helpers.NodeDrainerTestContext - var kubeSystemPods, immediatePods, allowCompletionPods, deleteTimeoutPods []string - var finalizerPod string +// TestDrainControllerBasicFlow tests the DrainController's basic drain flow. +func TestDrainControllerBasicFlow(t *testing.T) { + feature := features.New("TestDrainControllerBasicFlow"). + WithLabel("suite", "drain-controller") feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client, err := c.NewClient() - require.NoError(t, err) - - var newCtx context.Context - newCtx, testCtx = helpers.SetupNodeDrainerTest(ctx, t, c, "data/nd-all-modes.yaml", "immediate-test") + assert.NoError(t, err) - require.NoError(t, helpers.CreateNamespace(ctx, client, "allowcompletion-test")) - require.NoError(t, helpers.CreateNamespace(ctx, client, "delete-timeout-test")) + nodeName := helpers.SelectTestNodeFromUnusedPool(ctx, t, client) + t.Logf("Selected test node: %s", nodeName) - kubeSystemPods = helpers.CreatePodsFromTemplate(newCtx, t, client, "data/busybox-pods.yaml", testCtx.NodeName, "kube-system") - immediatePods = helpers.CreatePodsFromTemplate(newCtx, t, client, "data/busybox-pods.yaml", testCtx.NodeName, "immediate-test") - finalizerPodNames := helpers.CreatePodsFromTemplate(newCtx, t, client, "data/busybox-pod-with-finalizer.yaml", testCtx.NodeName, "immediate-test") - allowCompletionPods = helpers.CreatePodsFromTemplate(newCtx, t, client, "data/busybox-pods.yaml", testCtx.NodeName, "allowcompletion-test") - deleteTimeoutPods = helpers.CreatePodsFromTemplate(newCtx, t, client, "data/busybox-pods.yaml", testCtx.NodeName, "delete-timeout-test") + workloadNamespace := "drain-test" + err = helpers.CreateNamespace(ctx, client, workloadNamespace) + require.NoError(t, err) - require.Len(t, finalizerPodNames, 1) - finalizerPod = finalizerPodNames[0] + // Create test pods on the node + podTemplate := helpers.NewGPUPodSpec(workloadNamespace, 1) + helpers.CreatePodsAndWaitTillRunning(ctx, t, client, []string{nodeName}, podTemplate) - helpers.WaitForPodsRunning(newCtx, t, client, "kube-system", kubeSystemPods) - helpers.WaitForPodsRunning(newCtx, t, client, "immediate-test", append(immediatePods, finalizerPod)) - helpers.WaitForPodsRunning(newCtx, t, client, "allowcompletion-test", allowCompletionPods) - helpers.WaitForPodsRunning(newCtx, t, client, "delete-timeout-test", deleteTimeoutPods) + helpers.DeleteAllHealthEventCRDs(ctx, t, client) - return newCtx + ctx = context.WithValue(ctx, keyNodeName, nodeName) + ctx = context.WithValue(ctx, keyNamespace, workloadNamespace) + return ctx }) - feature.Assess("all eviction modes in single drain cycle", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + feature.Assess("DrainController transitions Quarantined event to Draining", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + nodeName := ctx.Value(keyNodeName).(string) + client, err := c.NewClient() require.NoError(t, err) - defer func() { - var p v1.Pod - if err := client.Resources().Get(ctx, finalizerPod, "immediate-test", &p); err == nil { - p.Finalizers = []string{} - _ = client.Resources().Update(ctx, &p) - } - _ = client.Resources().Delete(ctx, &p) - }() + // Create a fatal event + event := helpers.NewHealthEventCRD(nodeName). + WithSource("e2e-test"). + WithCheckName("GpuXidError"). + WithFatal(true). + WithHealthy(false). + WithErrorCodes("79"). + WithMessage("XID error occurred"). + WithRecommendedAction(nvsentinelv1alpha1.ActionRestartVM). + Build() + + created := helpers.CreateHealthEventCRD(ctx, t, client, event) + t.Logf("Created HealthEvent: %s", created.Name) + + ctx = context.WithValue(ctx, keyHealthEventName, created.Name) - event := helpers.NewHealthEvent(testCtx.NodeName). - WithErrorCode("79"). - WithMessage("GPU Fallen off the bus") - helpers.SendHealthEvent(ctx, t, event) + // Wait for QuarantineController to process first + helpers.WaitForHealthEventPhase(ctx, t, client, created.Name, nvsentinelv1alpha1.PhaseQuarantined) - helpers.WaitForNodeLabel(ctx, t, client, testCtx.NodeName, statemanager.NVSentinelStateLabelKey, helpers.DrainingLabelValue) + // Wait for DrainController to start draining + helpers.WaitForHealthEventPhase(ctx, t, client, created.Name, nvsentinelv1alpha1.PhaseDraining) + t.Log("DrainController started draining (phase=Draining)") - t.Log("Phase 1: Immediate mode evicts pods immediately") - helpers.WaitForPodsDeleted(ctx, t, client, "immediate-test", immediatePods) + return ctx + }) - t.Log("Phase 1: kube-system pods NOT evicted (namespace exclusion)") - helpers.AssertPodsNeverDeleted(ctx, t, client, "kube-system", kubeSystemPods) + feature.Assess("DrainController transitions to Drained after pods evicted", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + eventName := ctx.Value(keyHealthEventName).(string) + namespaceName := ctx.Value(keyNamespace).(string) - t.Log("Phase 1: Finalizer pod stuck in Terminating") - err = helpers.DeletePod(ctx, t, client, "immediate-test", finalizerPod, false) + client, err := c.NewClient() require.NoError(t, err) - require.Eventually(t, func() bool { - var p v1.Pod - err := client.Resources().Get(ctx, finalizerPod, "immediate-test", &p) - if err != nil { - return false - } - return p.DeletionTimestamp != nil - }, helpers.EventuallyWaitTimeout, helpers.WaitInterval) - - require.Never(t, func() bool { - var p v1.Pod - err := client.Resources().Get(ctx, finalizerPod, "immediate-test", &p) - return err != nil - }, helpers.NeverWaitTimeout, helpers.WaitInterval) - - t.Log("Phase 2: Both allowCompletion and deleteAfterTimeout waiting (verify for 15s)") - require.Never(t, func() bool { - for _, podName := range allowCompletionPods { - pod := &v1.Pod{} - if err := client.Resources().Get(ctx, podName, "allowcompletion-test", pod); err != nil { - return true - } - } - for _, podName := range deleteTimeoutPods { - pod := &v1.Pod{} - if err := client.Resources().Get(ctx, podName, "delete-timeout-test", pod); err != nil { - return true - } - } - return false - }, helpers.NeverWaitTimeout, helpers.WaitInterval, "both mode pods should wait, not be deleted immediately") - - t.Log("Phase 3: Waiting for deleteAfterTimeout to expire (~60s)") - // The deleteAfterTimeoutMinutes is set to 1 minute in nd-all-modes.yaml - // Adding 10s buffer to account for processing time - time.Sleep(70 * time.Second) - - t.Log("Phase 4: DeleteAfterTimeout pods force-deleted after timeout") - // This verifies that DeleteAfterTimeout is processed before AllowCompletion (not blocked by it) - helpers.WaitForPodsDeleted(ctx, t, client, "delete-timeout-test", deleteTimeoutPods) - - t.Log("Phase 5: Verify AllowCompletion pods are still waiting (priority verification)") - // AllowCompletion pods should still exist - they wait indefinitely for natural completion - for _, podName := range allowCompletionPods { - pod := &v1.Pod{} - err := client.Resources().Get(ctx, podName, "allowcompletion-test", pod) - require.NoError(t, err, "AllowCompletion pod %s should still exist after DeleteAfterTimeout completes", podName) - require.Nil(t, pod.DeletionTimestamp, "AllowCompletion pod %s should not be terminating", podName) - } + // Manually delete pods to simulate eviction completion + t.Log("Manually draining pods to simulate eviction") + helpers.DrainRunningPodsInNamespace(ctx, t, client, namespaceName) + + // Wait for DrainController to complete drain + event := helpers.WaitForHealthEventPhase(ctx, t, client, eventName, nvsentinelv1alpha1.PhaseDrained) + t.Logf("DrainController completed drain (phase=%s)", event.Status.Phase) + + // Verify PodsDrained condition is set + helpers.AssertPodsDrainedCondition(t, event) - t.Log("Phase 6: Manually completing AllowCompletion pods to finish drain") - helpers.DeletePodsByNames(ctx, t, client, "allowcompletion-test", allowCompletionPods) - helpers.WaitForPodsDeleted(ctx, t, client, "allowcompletion-test", allowCompletionPods) + return ctx + }) + + feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client, err := c.NewClient() + assert.NoError(t, err) - helpers.WaitForNodeLabel(ctx, t, client, testCtx.NodeName, statemanager.NVSentinelStateLabelKey, helpers.DrainSucceededLabelValue) + nodeName := ctx.Value(keyNodeName).(string) + namespaceName := ctx.Value(keyNamespace).(string) - helpers.DeletePodsByNames(ctx, t, client, "kube-system", kubeSystemPods) + // Uncordon node + node, err := helpers.GetNodeByName(ctx, client, nodeName) + if err == nil && node.Spec.Unschedulable { + node.Spec.Unschedulable = false + client.Resources().Update(ctx, node) + } + + helpers.DeleteNamespace(ctx, t, client, namespaceName) + helpers.DeleteAllHealthEventCRDs(ctx, t, client) return ctx }) - feature.Assess("drainer resumes work after restart", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + testEnv.Test(t, feature.Feature()) +} + +// TestDrainSkipOverride tests that drain can be skipped via override. +func TestDrainSkipOverride(t *testing.T) { + feature := features.New("TestDrainSkipOverride"). + WithLabel("suite", "drain-controller") + + feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client, err := c.NewClient() + assert.NoError(t, err) + + nodeName := helpers.SelectTestNodeFromUnusedPool(ctx, t, client) + t.Logf("Selected test node: %s", nodeName) + + workloadNamespace := "drain-skip-test" + err = helpers.CreateNamespace(ctx, client, workloadNamespace) require.NoError(t, err) - podNames := helpers.ResetNodeAndTriggerDrain(ctx, t, client, testCtx.NodeName, "allowcompletion-test") + // Create test pods on the node + podTemplate := helpers.NewGPUPodSpec(workloadNamespace, 1) + helpers.CreatePodsAndWaitTillRunning(ctx, t, client, []string{nodeName}, podTemplate) + + helpers.DeleteAllHealthEventCRDs(ctx, t, client) + + ctx = context.WithValue(ctx, keyNodeName, nodeName) + ctx = context.WithValue(ctx, keyNamespace, workloadNamespace) + return ctx + }) + + feature.Assess("Event with skip drain override skips drain phase", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + nodeName := ctx.Value(keyNodeName).(string) + namespaceName := ctx.Value(keyNamespace).(string) - restartTime := time.Now() - err = helpers.RestartDeployment(ctx, t, client, "node-drainer", helpers.NVSentinelNamespace) + client, err := c.NewClient() require.NoError(t, err) - require.Eventually(t, func() bool { - found, event := helpers.CheckNodeEventExists(ctx, client, testCtx.NodeName, "NodeDraining", "", restartTime) - if found { - t.Logf("Found event after restart: %s", event.Reason) + // Get current pod names + pods, err := helpers.GetPodsOnNode(ctx, client.Resources(), nodeName) + require.NoError(t, err) + var podNames []string + for _, pod := range pods { + if pod.Namespace == namespaceName { + podNames = append(podNames, pod.Name) } - return found - }, helpers.EventuallyWaitTimeout, helpers.WaitInterval) + } + + // Create event with skip drain override + event := helpers.NewHealthEventCRD(nodeName). + WithSource("e2e-test"). + WithCheckName("GpuXidError"). + WithFatal(true). + WithHealthy(false). + WithErrorCodes("79"). + WithSkipDrain(true). // Skip drain + Build() + + created := helpers.CreateHealthEventCRD(ctx, t, client, event) + t.Logf("Created HealthEvent with skip drain: %s", created.Name) - helpers.DeletePodsByNames(ctx, t, client, "allowcompletion-test", podNames) - helpers.WaitForPodsDeleted(ctx, t, client, "allowcompletion-test", podNames) - helpers.WaitForNodeLabel(ctx, t, client, testCtx.NodeName, statemanager.NVSentinelStateLabelKey, helpers.DrainSucceededLabelValue) + // Wait for quarantine (drain is skipped, but quarantine should still happen) + helpers.WaitForHealthEventPhase(ctx, t, client, created.Name, nvsentinelv1alpha1.PhaseQuarantined) + + // Verify event never reaches Draining phase + helpers.AssertHealthEventNeverReachesPhase(ctx, t, client, created.Name, nvsentinelv1alpha1.PhaseDraining) + + // Verify pods are NOT evicted + if len(podNames) > 0 { + helpers.AssertPodsNeverDeleted(ctx, t, client, namespaceName, podNames) + } return ctx }) feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client, err := c.NewClient() - require.NoError(t, err) + assert.NoError(t, err) + + nodeName := ctx.Value(keyNodeName).(string) + namespaceName := ctx.Value(keyNamespace).(string) + + // Uncordon node + node, err := helpers.GetNodeByName(ctx, client, nodeName) + if err == nil && node.Spec.Unschedulable { + node.Spec.Unschedulable = false + client.Resources().Update(ctx, node) + } - helpers.DeleteNamespace(ctx, t, client, "allowcompletion-test") - helpers.DeleteNamespace(ctx, t, client, "delete-timeout-test") + helpers.DeleteNamespace(ctx, t, client, namespaceName) + helpers.DeleteAllHealthEventCRDs(ctx, t, client) - return helpers.TeardownNodeDrainer(ctx, t, c) + return ctx }) testEnv.Test(t, feature.Feature()) } -func TestNodeDrainerPartialDrain(t *testing.T) { - feature := features.New("TestNodeDrainerPartialDrain"). - WithLabel("suite", "node-drainer") +// TestDrainWithKubeSystemExclusion tests that kube-system pods are not evicted. +func TestDrainWithKubeSystemExclusion(t *testing.T) { + feature := features.New("TestDrainWithKubeSystemExclusion"). + WithLabel("suite", "drain-controller") - var testCtx *helpers.NodeDrainerTestContext - var immediateEvictionPods []string - var immediateEvictionPodsWithImpactedGPU []string + var kubeSystemPodNames []string feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client, err := c.NewClient() - require.NoError(t, err) + assert.NoError(t, err) + + nodeName := helpers.SelectTestNodeFromUnusedPool(ctx, t, client) + t.Logf("Selected test node: %s", nodeName) - var newCtx context.Context - newCtx, testCtx = helpers.SetupNodeDrainerTest(ctx, t, c, "data/nd-all-modes.yaml", "immediate-test") - // Since SetupNodeDrainerTest will override the Tilt configmap, we need to ensure that the Helm chart value - // correctly set the value. - require.Contains(t, string(testCtx.ConfigMapBackup), "partialDrainEnabled = true") + // Record existing kube-system pods on this node + pods, err := helpers.GetPodsOnNode(ctx, client.Resources(), nodeName) + require.NoError(t, err) + for _, pod := range pods { + if pod.Namespace == "kube-system" && pod.Status.Phase == v1.PodRunning { + kubeSystemPodNames = append(kubeSystemPodNames, pod.Name) + } + } + t.Logf("Found %d kube-system pods on node %s", len(kubeSystemPodNames), nodeName) - // prevents conflicting processing from the fault-remediation module, specifically with it adding the - // dgxc.nvidia.com/nvsentinel-state label - err = helpers.ScaleDeployment(ctx, t, client, "fault-remediation", helpers.NVSentinelNamespace, 0) + // Create user workload namespace + workloadNamespace := "drain-exclusion-test" + err = helpers.CreateNamespace(ctx, client, workloadNamespace) require.NoError(t, err) - immediateEvictionPods = helpers.CreatePodsFromTemplate(newCtx, t, client, "data/busybox-pods.yaml", testCtx.NodeName, "immediate-test") - immediateEvictionPodsWithImpactedGPU = helpers.CreatePodsFromTemplate(newCtx, t, client, "data/busybox-pod-with-devices.yaml", testCtx.NodeName, "immediate-test") + // Create test pods + podTemplate := helpers.NewGPUPodSpec(workloadNamespace, 1) + helpers.CreatePodsAndWaitTillRunning(ctx, t, client, []string{nodeName}, podTemplate) - helpers.WaitForPodsRunning(newCtx, t, client, "immediate-test", append(immediateEvictionPods, - immediateEvictionPodsWithImpactedGPU...)) + helpers.DeleteAllHealthEventCRDs(ctx, t, client) - return newCtx + ctx = context.WithValue(ctx, keyNodeName, nodeName) + ctx = context.WithValue(ctx, keyNamespace, workloadNamespace) + return ctx }) - feature.Assess("Execute partial drain", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + + feature.Assess("kube-system pods are not evicted during drain", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + nodeName := ctx.Value(keyNodeName).(string) + namespaceName := ctx.Value(keyNamespace).(string) + client, err := c.NewClient() require.NoError(t, err) - event := helpers.NewHealthEvent(testCtx.NodeName). - WithErrorCode("119"). - WithMessage("NVRM: Xid (PCI:0002:00:00): 119, pid=1582259, name=nvc:[driver]"). - WithRecommendedAction(2). - WithEntitiesImpacted([]helpers.EntityImpacted{ - { - EntityType: "GPU_UUID", - EntityValue: "GPU-123", - }, - }) - helpers.SendHealthEvent(ctx, t, event) - helpers.WaitForNodeLabel(ctx, t, client, testCtx.NodeName, statemanager.NVSentinelStateLabelKey, helpers.DrainingLabelValue) - - t.Log("Phase 1: ImmediateEviction pod leveraging GPU is evicted immediately") - helpers.WaitForPodsDeleted(ctx, t, client, "immediate-test", immediateEvictionPodsWithImpactedGPU) - - t.Log("Phase 2: Draining succeeds after pod leveraging GPU is evicted") - helpers.WaitForNodeLabel(ctx, t, client, testCtx.NodeName, statemanager.NVSentinelStateLabelKey, helpers.DrainSucceededLabelValue) - - t.Log("Phase 3: ImmediateEviction pods not leveraging GPU should not be deleted") - helpers.AssertPodsNeverDeleted(ctx, t, client, "immediate-test", immediateEvictionPods) - - // remaining immediateEvictionPods will be deleted at the end of the following skip draining test + // Create fatal event + event := helpers.NewHealthEventCRD(nodeName). + WithSource("e2e-test"). + WithCheckName("GpuXidError"). + WithFatal(true). + WithHealthy(false). + WithErrorCodes("79"). + Build() + + created := helpers.CreateHealthEventCRD(ctx, t, client, event) + + // Wait for drain to start + helpers.WaitForHealthEventPhase(ctx, t, client, created.Name, nvsentinelv1alpha1.PhaseDraining) + + // Verify kube-system pods are NOT deleted + if len(kubeSystemPodNames) > 0 { + helpers.AssertPodsNeverDeleted(ctx, t, client, "kube-system", kubeSystemPodNames) + t.Logf("Verified %d kube-system pods were not evicted", len(kubeSystemPodNames)) + } + + // Manually drain user workload to complete the drain + helpers.DrainRunningPodsInNamespace(ctx, t, client, namespaceName) + + // Wait for drain to complete + helpers.WaitForHealthEventPhase(ctx, t, client, created.Name, nvsentinelv1alpha1.PhaseDrained) + return ctx }) - // We can verify if draining was skipped by leveraging the dgxc.nvidia.com/nvsentinel-state label. If draining - // is skipped, no label updates are applied so the value will retain its current value from the beginning of the drain - // evaluation and will not go from initialVal -> draining -> drain-succeeded or from initialVal -> drain-succeeded. - // To ensure that draining is skipped (the drain isn't re-processed and goes to StatusSucceeded) and that draining - // completes (the drain goes to status AlreadyDrained and no bugs prevent processing), we will manually add a - // value to the label other than drain-succeeded and ensure that the fault-remediation-module processes the event. - // This test ensures that the label goes from -> fault-remediated when draining is skipped which - // covers the 2 conditions above. - feature.Assess("Skip partial drain if pods using unhealthy GPU are already drained", func(ctx context.Context, - t *testing.T, c *envconf.Config) context.Context { + + feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client, err := c.NewClient() - require.NoError(t, err) - err = helpers.ScaleDeployment(ctx, t, client, "fault-remediation", helpers.NVSentinelNamespace, 1) - require.NoError(t, err) - helpers.WaitForDeploymentRollout(ctx, t, client, "fault-remediation", helpers.NVSentinelNamespace) + assert.NoError(t, err) - nodeLabelSequenceObserved := make(chan bool) - desiredNVSentinelStateNodeLabels := []string{ - string("placeholder"), - string(statemanager.RemediatingLabelValue), - string(statemanager.RemediationSucceededLabelValue), - } - err = helpers.StartNodeLabelWatcher(ctx, t, client, testCtx.NodeName, desiredNVSentinelStateNodeLabels, - false, nodeLabelSequenceObserved) - require.NoError(t, err) - err = helpers.SetNodeLabel(ctx, client, testCtx.NodeName, string(statemanager.NVSentinelStateLabelKey), "placeholder") - require.NoError(t, err) + nodeName := ctx.Value(keyNodeName).(string) + namespaceName := ctx.Value(keyNamespace).(string) - event := helpers.NewHealthEvent(testCtx.NodeName). - WithErrorCode("119"). - WithMessage("NVRM: Xid (PCI:0002:00:00): 119, pid=1582259, name=nvc:[driver]"). - WithRecommendedAction(2). - WithEntitiesImpacted([]helpers.EntityImpacted{ - { - EntityType: "GPU_UUID", - EntityValue: "GPU-123", - }, - }) - helpers.SendHealthEvent(ctx, t, event) - - t.Log("Phase 1: Drain should be skipped") - timer := time.NewTimer(1 * time.Minute) - - select { - case success := <-nodeLabelSequenceObserved: - require.True(t, success) - case <-timer.C: - require.Fail(t, "timed out waiting desired label changes") + // Uncordon node + node, err := helpers.GetNodeByName(ctx, client, nodeName) + if err == nil && node.Spec.Unschedulable { + node.Spec.Unschedulable = false + client.Resources().Update(ctx, node) } - t.Log("Phase 2: Pods not leveraging impacted GPU should not be drained") - helpers.AssertPodsNeverDeleted(ctx, t, client, "immediate-test", immediateEvictionPods) + helpers.DeleteNamespace(ctx, t, client, namespaceName) + helpers.DeleteAllHealthEventCRDs(ctx, t, client) - t.Log("Phase 3: Manually deleting pods not leveraging impacted GPU") - helpers.DeletePodsByNames(ctx, t, client, "immediate-test", immediateEvictionPods) + return ctx + }) + + testEnv.Test(t, feature.Feature()) +} + +// TestDrainPhaseSequence tests the full phase sequence through drain. +func TestDrainPhaseSequence(t *testing.T) { + feature := features.New("TestDrainPhaseSequence"). + WithLabel("suite", "drain-controller") + + feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client, err := c.NewClient() + assert.NoError(t, err) + + nodeName := helpers.SelectTestNodeFromUnusedPool(ctx, t, client) + t.Logf("Selected test node: %s", nodeName) + + workloadNamespace := "drain-sequence-test" + err = helpers.CreateNamespace(ctx, client, workloadNamespace) + require.NoError(t, err) + + podTemplate := helpers.NewGPUPodSpec(workloadNamespace, 1) + helpers.CreatePodsAndWaitTillRunning(ctx, t, client, []string{nodeName}, podTemplate) + helpers.DeleteAllHealthEventCRDs(ctx, t, client) + + ctx = context.WithValue(ctx, keyNodeName, nodeName) + ctx = context.WithValue(ctx, keyNamespace, workloadNamespace) return ctx }) - feature.Assess("Partial drain failure from GPU UUID missing", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + + feature.Assess("HealthEvent progresses through New → Quarantined → Draining → Drained", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + nodeName := ctx.Value(keyNodeName).(string) + namespaceName := ctx.Value(keyNamespace).(string) + client, err := c.NewClient() require.NoError(t, err) - event := helpers.NewHealthEvent(testCtx.NodeName). - WithErrorCode("119"). - WithMessage("NVRM: Xid (PCI:0002:00:00): 119, pid=1582259, name=nvc:[driver]"). - WithRecommendedAction(2) + // Create fatal event + event := helpers.NewHealthEventCRD(nodeName). + WithSource("e2e-test"). + WithCheckName("GpuXidError"). + WithFatal(true). + WithHealthy(false). + WithErrorCodes("79"). + Build() + + created := helpers.CreateHealthEventCRD(ctx, t, client, event) + t.Logf("Created HealthEvent: %s", created.Name) + + // Define expected phase sequence + sequence := helpers.ExpectedPhaseSequence{ + nvsentinelv1alpha1.PhaseQuarantined, + nvsentinelv1alpha1.PhaseDraining, + } + + // Wait for sequence up to Draining + helpers.WaitForHealthEventPhaseSequence(ctx, t, client, created.Name, sequence) + + // Manually drain to complete + helpers.DrainRunningPodsInNamespace(ctx, t, client, namespaceName) - helpers.SendHealthEvent(ctx, t, event) + // Wait for Drained + helpers.WaitForHealthEventPhase(ctx, t, client, created.Name, nvsentinelv1alpha1.PhaseDrained) - t.Log("Phase 1: Wait for node to have drain-failed label") - helpers.WaitForNodeLabel(ctx, t, client, testCtx.NodeName, statemanager.NVSentinelStateLabelKey, - string(statemanager.DrainFailedLabelValue)) + t.Log("Successfully verified phase sequence: New → Quarantined → Draining → Drained") return ctx }) + feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { client, err := c.NewClient() - require.NoError(t, err) + assert.NoError(t, err) + + nodeName := ctx.Value(keyNodeName).(string) + namespaceName := ctx.Value(keyNamespace).(string) - helpers.DeleteNamespace(ctx, t, client, "immediate-test") + // Uncordon node + node, err := helpers.GetNodeByName(ctx, client, nodeName) + if err == nil && node.Spec.Unschedulable { + node.Spec.Unschedulable = false + client.Resources().Update(ctx, node) + } + + helpers.DeleteNamespace(ctx, t, client, namespaceName) + helpers.DeleteAllHealthEventCRDs(ctx, t, client) - return helpers.TeardownNodeDrainer(ctx, t, c) + return ctx }) testEnv.Test(t, feature.Feature()) diff --git a/tests/smoke_test.go b/tests/smoke_test.go index 04dab6de4..01aed1011 100644 --- a/tests/smoke_test.go +++ b/tests/smoke_test.go @@ -20,15 +20,13 @@ package tests import ( "context" "testing" - "time" "github.com/stretchr/testify/assert" - v1 "k8s.io/api/core/v1" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" "tests/helpers" - "github.com/nvidia/nvsentinel/commons/pkg/statemanager" + nvsentinelv1alpha1 "github.com/nvidia/nvsentinel/api/nvsentinel/v1alpha1" ) func TestFatalHealthEvent(t *testing.T) { @@ -41,9 +39,6 @@ func TestFatalHealthEvent(t *testing.T) { client, err := c.NewClient() assert.NoError(t, err, "failed to create kubernetes client") - ctx = helpers.ApplyQuarantineConfig(ctx, t, c, "data/basic-matching-configmap.yaml") - ctx = helpers.ApplyNodeDrainerConfig(ctx, t, c, "data/nd-all-modes.yaml") - // Use a real (non-KWOK) node for smoke test to validate actual container execution nodeName, err := helpers.GetRealNodeName(ctx, client) assert.NoError(t, err, "failed to get real node") @@ -55,43 +50,53 @@ func TestFatalHealthEvent(t *testing.T) { podTemplate := helpers.NewGPUPodSpec(workloadNamespace, 1) helpers.CreatePodsAndWaitTillRunning(ctx, t, client, []string{nodeName}, podTemplate) + // Clean up any existing HealthEvents for this node + helpers.DeleteAllHealthEventCRDs(ctx, t, client) + ctx = context.WithValue(ctx, keyNodeName, nodeName) ctx = context.WithValue(ctx, keyNamespace, workloadNamespace) return ctx }) - nodeLabelSequenceObserved := make(chan bool) - feature.Assess("Can start node label watcher", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + feature.Assess("Can create fatal HealthEvent CRD", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + nodeName := ctx.Value(keyNodeName).(string) + client, err := c.NewClient() assert.NoError(t, err, "failed to create kubernetes client") - nodeName := ctx.Value(keyNodeName).(string) - t.Logf("Starting label sequence watcher for node %s", nodeName) - desiredNVSentinelStateNodeLabels := []string{ - string(statemanager.QuarantinedLabelValue), - string(statemanager.DrainingLabelValue), - string(statemanager.DrainSucceededLabelValue), - string(statemanager.RemediatingLabelValue), - string(statemanager.RemediationSucceededLabelValue), - } - err = helpers.StartNodeLabelWatcher(ctx, t, client, nodeName, desiredNVSentinelStateNodeLabels, true, nodeLabelSequenceObserved) - assert.NoError(t, err, "failed to start node label watcher") - - // Sleep to ensure Kubernetes watch is fully established before triggering state changes - // This prevents missing early label transitions due to watch startup latency - t.Log("Waiting for watch to establish connection...") - time.Sleep(2 * time.Second) - t.Log("Watch established, ready for health event") - + // Create a fatal HealthEvent CRD (replaces HTTP POST to health-event endpoint) + event := helpers.NewHealthEventCRD(nodeName). + WithSource("e2e-smoke-test"). + WithCheckName("GpuXidError"). + WithComponentClass("GPU"). + WithFatal(true). + WithHealthy(false). + WithErrorCodes("79"). + WithEntity("GPU", "0"). + WithMessage("XID error occurred"). + WithRecommendedAction(nvsentinelv1alpha1.ActionRestartVM). + Build() + + created := helpers.CreateHealthEventCRD(ctx, t, client, event) + t.Logf("Created HealthEvent CRD: %s", created.Name) + + ctx = context.WithValue(ctx, keyHealthEventName, created.Name) return ctx }) - feature.Assess("Can send fatal health event", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - nodeName := ctx.Value(keyNodeName).(string) + feature.Assess("HealthEvent transitions to Quarantined phase", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + eventName := ctx.Value(keyHealthEventName).(string) + + client, err := c.NewClient() + assert.NoError(t, err, "failed to create kubernetes client") + + // Wait for QuarantineController to process the event + event := helpers.WaitForHealthEventPhase(ctx, t, client, eventName, nvsentinelv1alpha1.PhaseQuarantined) + t.Logf("HealthEvent %s reached phase: %s", eventName, event.Status.Phase) - err := helpers.SendHealthEventsToNodes([]string{nodeName}, "data/fatal-health-event.json") - assert.NoError(t, err, "failed to send health event") + // Verify NodeQuarantined condition is set + helpers.AssertNodeQuarantinedCondition(t, event) return ctx }) @@ -108,70 +113,59 @@ func TestFatalHealthEvent(t *testing.T) { node, err := helpers.GetNodeByName(ctx, client, nodeName) assert.NoError(t, err, "failed to get node after cordoning") - assert.Equal(t, "NVSentinel", node.Labels["cordon-by"]) - assert.Equal(t, "Basic-Match-Rule", node.Labels["cordon-reason"]) - - var nodeCondition *v1.NodeCondition - for i := range node.Status.Conditions { - if node.Status.Conditions[i].Type == "GpuXidError" { - nodeCondition = &node.Status.Conditions[i] - break - } - } - assert.NotNil(t, nodeCondition, "node condition GpuXidError not found") - - assert.Equal(t, "GpuXidError", string(nodeCondition.Type)) - assert.Equal(t, "True", string(nodeCondition.Status)) - assert.Equal(t, "GpuXidErrorIsNotHealthy", nodeCondition.Reason) - assert.Equal(t, "ErrorCode:79 GPU:0 XID error occurred Recommended Action=RESTART_VM;", nodeCondition.Message) + // Verify node is unschedulable + assert.True(t, node.Spec.Unschedulable, "node should be unschedulable (cordoned)") return ctx }) - feature.Assess("Drain label is set and pods are not evicted, delete the pod to move the process forward", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - nodeName := ctx.Value(keyNodeName).(string) + feature.Assess("HealthEvent transitions to Draining then Drained", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + eventName := ctx.Value(keyHealthEventName).(string) namespaceName := ctx.Value(keyNamespace).(string) client, err := c.NewClient() assert.NoError(t, err, "failed to create kubernetes client") - helpers.WaitForNodesWithLabel(ctx, t, client, []string{nodeName}, statemanager.NVSentinelStateLabelKey, string(statemanager.DrainingLabelValue)) - - expectedDrainingNodeEvent := v1.Event{ - Type: "NodeDraining", - Reason: "AwaitingPodCompletion", - } - helpers.WaitForNodeEvent(ctx, t, client, nodeName, expectedDrainingNodeEvent) + // Wait for DrainController to start draining + helpers.WaitForHealthEventPhase(ctx, t, client, eventName, nvsentinelv1alpha1.PhaseDraining) + t.Logf("HealthEvent %s is in Draining phase", eventName) + // Manually drain pods to simulate completion (for test with AllowCompletion mode) helpers.DrainRunningPodsInNamespace(ctx, t, client, namespaceName) + // Wait for drain to complete + event := helpers.WaitForHealthEventPhase(ctx, t, client, eventName, nvsentinelv1alpha1.PhaseDrained) + t.Logf("HealthEvent %s reached phase: %s", eventName, event.Status.Phase) + + // Verify PodsDrained condition is set + helpers.AssertPodsDrainedCondition(t, event) + return ctx }) - feature.Assess("Remediation CR is created and completes", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + feature.Assess("HealthEvent transitions to Remediated", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + eventName := ctx.Value(keyHealthEventName).(string) nodeName := ctx.Value(keyNodeName).(string) client, err := c.NewClient() assert.NoError(t, err, "failed to create kubernetes client") - rebootNode := helpers.WaitForRebootNodeCR(ctx, t, client, nodeName) + // Wait for RemediationController to process the event + // This will create a RebootNode CR and wait for it to complete + event := helpers.WaitForHealthEventPhase(ctx, t, client, eventName, nvsentinelv1alpha1.PhaseRemediated) + t.Logf("HealthEvent %s reached phase: %s", eventName, event.Status.Phase) + // Verify Remediated condition is set + helpers.AssertRemediatedCondition(t, event) + + // Clean up the RebootNode CR created by remediation + rebootNode := helpers.WaitForRebootNodeCR(ctx, t, client, nodeName) err = helpers.DeleteRebootNodeCR(ctx, client, rebootNode) assert.NoError(t, err, "failed to delete RebootNode CR") return ctx }) - feature.Assess("Audit logs are generated for API calls", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - client, err := c.NewClient() - assert.NoError(t, err, "failed to create kubernetes client") - - components := []string{"fault-quarantine", "fault-remediation", "janitor"} - helpers.VerifyAuditLogsExist(ctx, t, client, components) - - return ctx - }) - feature.Assess("Log-collector job completes successfully", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { nodeName := ctx.Value(keyNodeName).(string) @@ -189,11 +183,31 @@ func TestFatalHealthEvent(t *testing.T) { return ctx }) - feature.Assess("Can send healthy event", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + feature.Assess("Create healthy HealthEvent to resolve", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { nodeName := ctx.Value(keyNodeName).(string) - err := helpers.SendHealthEventsToNodes([]string{nodeName}, "data/healthy-event.json") - assert.NoError(t, err, "failed to send health event") + client, err := c.NewClient() + assert.NoError(t, err, "failed to create kubernetes client") + + // Create a healthy event (replaces HTTP POST to health-event endpoint) + helpers.SendHealthyEventViaCRD(ctx, t, client, nodeName) + t.Logf("Created healthy HealthEvent for node %s", nodeName) + + return ctx + }) + + feature.Assess("Original HealthEvent transitions to Resolved", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + eventName := ctx.Value(keyHealthEventName).(string) + + client, err := c.NewClient() + assert.NoError(t, err, "failed to create kubernetes client") + + // Wait for the original event to be resolved + event := helpers.WaitForHealthEventPhase(ctx, t, client, eventName, nvsentinelv1alpha1.PhaseResolved) + t.Logf("HealthEvent %s reached phase: %s", eventName, event.Status.Phase) + + // Verify ResolvedAt timestamp is set + helpers.AssertResolvedAtSet(t, event) return ctx }) @@ -207,44 +221,12 @@ func TestFatalHealthEvent(t *testing.T) { t.Logf("Waiting for node %s to be uncordoned", nodeName) helpers.WaitForNodesCordonState(ctx, t, client, []string{nodeName}, false) - // Wait for node condition to be updated to healthy - t.Logf("Waiting for node %s condition to become healthy", nodeName) - helpers.WaitForNodeConditionWithCheckName(ctx, t, client, nodeName, "GpuXidError", "", "GpuXidErrorIsHealthy", v1.ConditionFalse) - node, err := helpers.GetNodeByName(ctx, client, nodeName) assert.NoError(t, err, "failed to get node after uncordoning") - assert.Equal(t, "NVSentinel", node.Labels["uncordon-by"]) - - var nodeCondition *v1.NodeCondition - for i := range node.Status.Conditions { - if node.Status.Conditions[i].Type == "GpuXidError" { - nodeCondition = &node.Status.Conditions[i] - break - } - } - assert.NotNil(t, nodeCondition, "node condition GpuXidError not found") - - assert.Equal(t, "GpuXidError", string(nodeCondition.Type)) - assert.Equal(t, "False", string(nodeCondition.Status)) - assert.Equal(t, "GpuXidErrorIsHealthy", nodeCondition.Reason) - assert.Equal(t, "No Health Failures", nodeCondition.Message) - - return ctx - }) + // Verify node is schedulable again + assert.False(t, node.Spec.Unschedulable, "node should be schedulable (uncordoned)") - feature.Assess("Observed NVSentinel expected state label changes", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - // With state buffering (1000ms window) and out-of-order sorting implemented in - // StartNodeLabelWatcher, we can now reliably observe rapid state transitions that occur - // with PostgreSQL LISTEN/NOTIFY. The watcher buffers updates and sorts them by expected - // sequence to capture all intermediate states even when Kubernetes watch delivers updates - // out of order or with significant latency. - select { - case success := <-nodeLabelSequenceObserved: - assert.True(t, success) - default: - assert.Fail(t, "did not observe expected label changes for nvsentinel-state") - } return ctx }) @@ -256,8 +238,8 @@ func TestFatalHealthEvent(t *testing.T) { err = helpers.DeleteNamespace(ctx, t, client, namespaceName) assert.NoError(t, err, "failed to delete workloads namespace") - helpers.RestoreQuarantineConfig(ctx, t, c) - helpers.RestoreNodeDrainerConfig(ctx, t, c) + // Clean up HealthEvent CRDs + helpers.DeleteAllHealthEventCRDs(ctx, t, client) return ctx }) @@ -275,9 +257,6 @@ func TestFatalUnsupportedHealthEvent(t *testing.T) { client, err := c.NewClient() assert.NoError(t, err, "failed to create kubernetes client") - ctx = helpers.ApplyQuarantineConfig(ctx, t, c, "data/basic-matching-configmap.yaml") - ctx = helpers.ApplyNodeDrainerConfig(ctx, t, c, "data/nd-all-modes.yaml") - // Use a real (non-KWOK) node for smoke test to validate actual container execution nodeName, err := helpers.GetRealNodeName(ctx, client) assert.NoError(t, err, "failed to get real node") @@ -289,23 +268,50 @@ func TestFatalUnsupportedHealthEvent(t *testing.T) { podTemplate := helpers.NewGPUPodSpec(workloadNamespace, 1) helpers.CreatePodsAndWaitTillRunning(ctx, t, client, []string{nodeName}, podTemplate) + // Clean up any existing resources + helpers.DeleteAllHealthEventCRDs(ctx, t, client) + helpers.DeleteAllLogCollectorJobs(ctx, t, client) + ctx = context.WithValue(ctx, keyNodeName, nodeName) ctx = context.WithValue(ctx, keyNamespace, workloadNamespace) return ctx }) - feature.Assess("Can send fatal health event", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + feature.Assess("Can create unsupported fatal HealthEvent CRD", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { nodeName := ctx.Value(keyNodeName).(string) client, err := c.NewClient() assert.NoError(t, err, "failed to create kubernetes client") - // Delete any existing log-collector jobs for this node before sending event - helpers.DeleteAllLogCollectorJobs(ctx, t, client) + // Create an unsupported fatal HealthEvent (XID 145 = CONTACT_SUPPORT) + event := helpers.NewHealthEventCRD(nodeName). + WithSource("e2e-smoke-test"). + WithCheckName("GpuXidError"). + WithComponentClass("GPU"). + WithFatal(true). + WithHealthy(false). + WithErrorCodes("145"). + WithEntity("GPU", "0"). + WithMessage("XID error occurred"). + WithRecommendedAction(nvsentinelv1alpha1.ActionContactSupport). + Build() + + created := helpers.CreateHealthEventCRD(ctx, t, client, event) + t.Logf("Created HealthEvent CRD: %s", created.Name) + + ctx = context.WithValue(ctx, keyHealthEventName, created.Name) + return ctx + }) + + feature.Assess("HealthEvent transitions to Quarantined phase", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + eventName := ctx.Value(keyHealthEventName).(string) + + client, err := c.NewClient() + assert.NoError(t, err, "failed to create kubernetes client") - err = helpers.SendHealthEventsToNodes([]string{nodeName}, "data/unsupported-health-event.json") - assert.NoError(t, err, "failed to send health event") + event := helpers.WaitForHealthEventPhase(ctx, t, client, eventName, nvsentinelv1alpha1.PhaseQuarantined) + t.Logf("HealthEvent %s reached phase: %s", eventName, event.Status.Phase) return ctx }) @@ -319,50 +325,25 @@ func TestFatalUnsupportedHealthEvent(t *testing.T) { t.Logf("Waiting for node %s to be cordoned", nodeName) helpers.WaitForNodesCordonState(ctx, t, client, []string{nodeName}, true) - // Wait for node condition to be updated to unhealthy - t.Logf("Waiting for node %s condition to become unhealthy", nodeName) - helpers.WaitForNodeConditionWithCheckName(ctx, t, client, nodeName, "GpuXidError", "", "GpuXidErrorIsNotHealthy", v1.ConditionTrue) - - node, err := helpers.GetNodeByName(ctx, client, nodeName) - assert.NoError(t, err, "failed to get node after cordoning") - - assert.Equal(t, "NVSentinel", node.Labels["cordon-by"]) - assert.Equal(t, "Basic-Match-Rule", node.Labels["cordon-reason"]) - - var nodeCondition *v1.NodeCondition - for i := range node.Status.Conditions { - if node.Status.Conditions[i].Type == "GpuXidError" { - nodeCondition = &node.Status.Conditions[i] - break - } - } - assert.NotNil(t, nodeCondition, "node condition GpuXidError not found") - - assert.Equal(t, "GpuXidError", string(nodeCondition.Type)) - assert.Equal(t, "True", string(nodeCondition.Status)) - assert.Equal(t, "GpuXidErrorIsNotHealthy", nodeCondition.Reason) - assert.Equal(t, "ErrorCode:145 GPU:0 XID error occurred Recommended Action=CONTACT_SUPPORT;", nodeCondition.Message) - return ctx }) - feature.Assess("Drain label is set and pods are not evicted, delete the pod to move the process forward", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - nodeName := ctx.Value(keyNodeName).(string) + feature.Assess("HealthEvent transitions through Draining to Drained", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + eventName := ctx.Value(keyHealthEventName).(string) namespaceName := ctx.Value(keyNamespace).(string) client, err := c.NewClient() assert.NoError(t, err, "failed to create kubernetes client") - helpers.WaitForNodesWithLabel(ctx, t, client, []string{nodeName}, statemanager.NVSentinelStateLabelKey, string(statemanager.DrainingLabelValue)) - - expectedDrainingNodeEvent := v1.Event{ - Type: "NodeDraining", - Reason: "AwaitingPodCompletion", - } - helpers.WaitForNodeEvent(ctx, t, client, nodeName, expectedDrainingNodeEvent) + // Wait for DrainController to start draining + helpers.WaitForHealthEventPhase(ctx, t, client, eventName, nvsentinelv1alpha1.PhaseDraining) + // Manually drain pods helpers.DrainRunningPodsInNamespace(ctx, t, client, namespaceName) + // Wait for drain to complete + helpers.WaitForHealthEventPhase(ctx, t, client, eventName, nvsentinelv1alpha1.PhaseDrained) + return ctx }) @@ -372,29 +353,33 @@ func TestFatalUnsupportedHealthEvent(t *testing.T) { client, err := c.NewClient() assert.NoError(t, err, "failed to create kubernetes client") - // For unsupported events, log-collector should NOT be triggered - // since shouldSkipEvent returns true and we return early before runLogCollector + // For unsupported events (CONTACT_SUPPORT), log-collector should NOT be triggered helpers.VerifyNoLogCollectorJobExists(ctx, t, client, nodeName) return ctx }) - feature.Assess("Remediation failed label is set", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - nodeName := ctx.Value(keyNodeName).(string) + feature.Assess("HealthEvent stays in Drained (remediation skipped for CONTACT_SUPPORT)", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + eventName := ctx.Value(keyHealthEventName).(string) client, err := c.NewClient() assert.NoError(t, err, "failed to create kubernetes client") - helpers.WaitForNodesWithLabel(ctx, t, client, []string{nodeName}, statemanager.NVSentinelStateLabelKey, string(statemanager.RemediationFailedLabelValue)) + // For CONTACT_SUPPORT events, the RemediationController should not progress to Remediated + // because no automatic remediation is possible + helpers.AssertHealthEventNeverReachesPhase(ctx, t, client, eventName, nvsentinelv1alpha1.PhaseRemediated) return ctx }) - feature.Assess("Can send healthy event", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + feature.Assess("Create healthy HealthEvent to resolve", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { nodeName := ctx.Value(keyNodeName).(string) - err := helpers.SendHealthEventsToNodes([]string{nodeName}, "data/healthy-event.json") - assert.NoError(t, err, "failed to send health event") + client, err := c.NewClient() + assert.NoError(t, err, "failed to create kubernetes client") + + helpers.SendHealthyEventViaCRD(ctx, t, client, nodeName) + t.Logf("Created healthy HealthEvent for node %s", nodeName) return ctx }) @@ -408,28 +393,10 @@ func TestFatalUnsupportedHealthEvent(t *testing.T) { t.Logf("Waiting for node %s to be uncordoned", nodeName) helpers.WaitForNodesCordonState(ctx, t, client, []string{nodeName}, false) - // Wait for node condition to be updated to healthy - t.Logf("Waiting for node %s condition to become healthy", nodeName) - helpers.WaitForNodeConditionWithCheckName(ctx, t, client, nodeName, "GpuXidError", "", "GpuXidErrorIsHealthy", v1.ConditionFalse) - node, err := helpers.GetNodeByName(ctx, client, nodeName) assert.NoError(t, err, "failed to get node after uncordoning") - assert.Equal(t, "NVSentinel", node.Labels["uncordon-by"]) - - var nodeCondition *v1.NodeCondition - for i := range node.Status.Conditions { - if node.Status.Conditions[i].Type == "GpuXidError" { - nodeCondition = &node.Status.Conditions[i] - break - } - } - assert.NotNil(t, nodeCondition, "node condition GpuXidError not found") - - assert.Equal(t, "GpuXidError", string(nodeCondition.Type)) - assert.Equal(t, "False", string(nodeCondition.Status)) - assert.Equal(t, "GpuXidErrorIsHealthy", nodeCondition.Reason) - assert.Equal(t, "No Health Failures", nodeCondition.Message) + assert.False(t, node.Spec.Unschedulable, "node should be schedulable (uncordoned)") return ctx }) @@ -442,8 +409,8 @@ func TestFatalUnsupportedHealthEvent(t *testing.T) { err = helpers.DeleteNamespace(ctx, t, client, namespaceName) assert.NoError(t, err, "failed to delete workloads namespace") - helpers.RestoreQuarantineConfig(ctx, t, c) - helpers.RestoreNodeDrainerConfig(ctx, t, c) + // Clean up HealthEvent CRDs + helpers.DeleteAllHealthEventCRDs(ctx, t, client) return ctx })