Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions cmd/dra-example-kubeletplugin/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,15 +53,22 @@ func NewDriver(ctx context.Context, config *Config) (*driver, error) {
}
driver.state = state

helper, err := kubeletplugin.Start(
ctx,
driver,
kubeletPluginOptions := []kubeletplugin.Option{
kubeletplugin.KubeClient(config.coreclient),
kubeletplugin.NodeName(config.flags.nodeName),
kubeletplugin.DriverName(consts.DriverName),
kubeletplugin.RegistrarDirectoryPath(config.flags.kubeletRegistrarDirectoryPath),
kubeletplugin.PluginDataDirectoryPath(config.DriverPluginPath()),
)
}

// Enable seamless upgrades when both seamless upgrades are enabled and POD_UID is available
if config.flags.seamlessUpgrades && config.flags.podUID != "" {
kubeletPluginOptions = append(kubeletPluginOptions,
kubeletplugin.RollingUpdate(types.UID(config.flags.podUID)),
)
}

helper, err := kubeletplugin.Start(ctx, driver, kubeletPluginOptions...)
if err != nil {
return nil, err
}
Expand Down
16 changes: 12 additions & 4 deletions cmd/dra-example-kubeletplugin/health.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,12 @@ func startHealthcheck(ctx context.Context, config *Config) (*healthcheck, error)

regSockPath := (&url.URL{
Scheme: "unix",
// TODO: this needs to adapt when seamless upgrades
// are enabled and the filename includes a uid.
Path: path.Join(config.flags.kubeletRegistrarDirectoryPath, consts.DriverName+"-reg.sock"),
Path: func() string {
if config.flags.seamlessUpgrades && config.flags.podUID != "" {
return path.Join(config.flags.kubeletRegistrarDirectoryPath, consts.DriverName+"-"+config.flags.podUID+"-reg.sock")
}
return path.Join(config.flags.kubeletRegistrarDirectoryPath, consts.DriverName+"-reg.sock")
}(),
}).String()
log.Info("connecting to registration socket", "path", regSockPath)
regConn, err := grpc.NewClient(
Expand All @@ -78,7 +81,12 @@ func startHealthcheck(ctx context.Context, config *Config) (*healthcheck, error)

draSockPath := (&url.URL{
Scheme: "unix",
Path: path.Join(config.DriverPluginPath(), "dra.sock"),
Path: func() string {
if config.flags.seamlessUpgrades && config.flags.podUID != "" {
return path.Join(config.DriverPluginPath(), "dra-"+config.flags.podUID+".sock")
}
return path.Join(config.DriverPluginPath(), "dra.sock")
}(),
}).String()
log.Info("connecting to DRA socket", "path", draSockPath)
draConn, err := grpc.NewClient(
Expand Down
14 changes: 14 additions & 0 deletions cmd/dra-example-kubeletplugin/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ type Flags struct {
kubeletRegistrarDirectoryPath string
kubeletPluginsDirectoryPath string
healthcheckPort int
podUID string
seamlessUpgrades bool
}

type Config struct {
Expand Down Expand Up @@ -115,6 +117,18 @@ func newApp() *cli.App {
Destination: &flags.healthcheckPort,
EnvVars: []string{"HEALTHCHECK_PORT"},
},
&cli.StringFlag{
Name: "pod-uid",
Usage: "UID of the pod (used for seamless upgrades to create unique socket names).",
Destination: &flags.podUID,
EnvVars: []string{"POD_UID"},
},
&cli.BoolFlag{
Name: "seamless-upgrades",
Usage: "Enable seamless upgrades support. When enabled, the driver will use rolling update mode if pod-uid is available.",
Destination: &flags.seamlessUpgrades,
EnvVars: []string{"SEAMLESS_UPGRADES"},
},
}
cliFlags = append(cliFlags, flags.kubeClientConfig.Flags()...)
cliFlags = append(cliFlags, flags.loggingConfig.Flags()...)
Expand Down
39 changes: 39 additions & 0 deletions deployments/helm/dra-example-driver/templates/kubeletplugin.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,38 @@ spec:
app.kubernetes.io/component: kubeletplugin
{{- with .Values.kubeletPlugin.updateStrategy }}
updateStrategy:
{{- if $.Values.kubeletPlugin.seamlessUpgrades.enabled }}
{{- $strategy := . }}
{{- range $key, $value := . }}
{{- if eq $key "rollingUpdate" }}
{{ $key }}:
maxSurge: 1
maxUnavailable: 0
{{- range $subkey, $subvalue := $value }}
{{- if and (ne $subkey "maxSurge") (ne $subkey "maxUnavailable") }}
{{ $subkey }}: {{ $subvalue }}
{{- end }}
{{- end }}
{{- else }}
{{ $key }}: {{ $value }}
{{- end }}
{{- end }}
{{- else }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- else }}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Setting kubeletPlugin.seamlessUpgrades.enabled will silently ignore the kubeletPlugin.updateStrategy value which isn't ideal.

Maybe we could keep the existing updateStrategy template block and use a different default for maxSurge and maxUnavailable when seamless upgrades are enabled? Then kubeletPlugin.updateStrategy is still authoritative if users express an opinion on that, but only setting kubeletPlugin.seamlessUpgrades.enabled is enough to set them to values that actually exercise seamless upgrades if they don't.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just to confirm, if user provides updateStrategy but seamlessUpgrades is enabled, then overwrite just the maxSurge and maxUnavailable of the given updateStrategy?

updateStrategy:
type: {{ .Values.kubeletPlugin.updateStrategy.type | default "RollingUpdate" }}
{{- if eq (.Values.kubeletPlugin.updateStrategy.type | default "RollingUpdate") "RollingUpdate" }}
rollingUpdate:
{{- if .Values.kubeletPlugin.seamlessUpgrades.enabled }}
maxSurge: 1
maxUnavailable: 0
{{- else }}
maxSurge: {{ .Values.kubeletPlugin.updateStrategy.rollingUpdate.maxSurge | default 0 }}
maxUnavailable: {{ .Values.kubeletPlugin.updateStrategy.rollingUpdate.maxUnavailable | default 1 }}
{{- end }}
{{- end }}
{{- end }}
template:
metadata:
Expand Down Expand Up @@ -80,6 +111,14 @@ spec:
- name: HEALTHCHECK_PORT
value: {{ .Values.kubeletPlugin.containers.plugin.healthcheckPort | quote }}
{{- end }}
- name: POD_UID
valueFrom:
fieldRef:
fieldPath: metadata.uid
{{- if .Values.kubeletPlugin.seamlessUpgrades.enabled }}
- name: SEAMLESS_UPGRADES
value: "true"
{{- end }}
volumeMounts:
- name: plugins-registry
mountPath: {{ .Values.kubeletPlugin.kubeletRegistrarDirectoryPath | quote }}
Expand Down
2 changes: 2 additions & 0 deletions deployments/helm/dra-example-driver/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ controller:
kubeletPlugin:
numDevices: 8
priorityClassName: "system-node-critical"
seamlessUpgrades:
enabled: true
updateStrategy:
type: RollingUpdate
podAnnotations: {}
Expand Down
Loading