diff --git a/deployments/container/Dockerfile b/deployments/container/Dockerfile
index ce55197cb..6dfdcf60a 100644
--- a/deployments/container/Dockerfile
+++ b/deployments/container/Dockerfile
@@ -33,8 +33,8 @@ RUN set -eux; \
     wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \
     | tar -C /usr/local -xz
 
-ENV GOPATH /go
-ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
+ENV GOPATH=/go
+ENV PATH=$GOPATH/bin:/usr/local/go/bin:$PATH
 
 WORKDIR /build
 COPY . .
@@ -50,22 +50,11 @@ FROM redhat/ubi9-minimal:latest AS minimal
 RUN rpm -qa --queryformat='^%{NAME}-\[0-9\].*\.%{ARCH}$\n' | sort -u > /tmp/package-names.minimal
 RUN rpm -qa | sort -u > /tmp/package-list.minimal
 
-# We define the following image as a base image and remove unneeded packages.
-FROM nvcr.io/nvidia/cuda:12.9.1-base-ubi9 AS base
+FROM nvcr.io/nvidia/distroless/go:v3.1.9-dev
 
-WORKDIR /cleanup
-
-COPY --from=minimal /tmp/package-names.minimal package-names.minimal
-COPY --from=minimal /tmp/package-list.minimal package-list.minimal
-COPY deployments/container/cleanup/* .
-
-RUN ./cleanup.sh
-
-WORKDIR /
-
-# We use the base images constructed above.
-# TODO: We will move to a shared base image once this implementation has been stabilized.
-FROM base
+USER 0:0
+SHELL ["/busybox/sh", "-c"]
+RUN ln -s /busybox/sh /bin/sh
 
 ENV NVIDIA_DISABLE_REQUIRE="true"
 ENV NVIDIA_VISIBLE_DEVICES=all
@@ -83,7 +72,7 @@ LABEL release="N/A"
 LABEL summary="NVIDIA device plugin for Kubernetes"
 LABEL description="See summary"
 
-RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
+COPY LICENSE /licenses/
 
 COPY --from=build /artifacts/config-manager         /usr/bin/config-manager
 COPY --from=build /artifacts/gpu-feature-discovery  /usr/bin/gpu-feature-discovery
diff --git a/deployments/container/Makefile b/deployments/container/Makefile
index 04a4b8cdf..aa00772b4 100644
--- a/deployments/container/Makefile
+++ b/deployments/container/Makefile
@@ -29,16 +29,16 @@ endif
 
 IMAGE_VERSION := $(VERSION)
 
-IMAGE_TAG ?= $(IMAGE_VERSION)-$(DIST)
+IMAGE_TAG ?= $(IMAGE_VERSION)
 IMAGE = $(IMAGE_NAME):$(IMAGE_TAG)
 
 OUT_IMAGE_NAME ?= $(IMAGE_NAME)
 OUT_IMAGE_VERSION ?= $(IMAGE_VERSION)
-OUT_IMAGE_TAG = $(OUT_IMAGE_VERSION)-$(DIST)
+OUT_IMAGE_TAG = $(OUT_IMAGE_VERSION)
 OUT_IMAGE = $(OUT_IMAGE_NAME):$(OUT_IMAGE_TAG)
 
 ##### Public rules #####
-DEFAULT_PUSH_TARGET := ubi9
+DEFAULT_PUSH_TARGET := image
 DISTRIBUTIONS = $(DEFAULT_PUSH_TARGET)
 
 IMAGE_TARGETS := $(patsubst %,image-%,$(DISTRIBUTIONS))
@@ -65,11 +65,8 @@ ifeq ($(PUSH_MULTIPLE_TAGS),true)
 push-$(DEFAULT_PUSH_TARGET): push-short
 endif
 
-push-%: DIST = $(*)
-push-short: DIST = $(DEFAULT_PUSH_TARGET)
 
-build-%: DIST = $(*)
-build-%: DOCKERFILE = $(CURDIR)/deployments/container/Dockerfile
+DOCKERFILE = $(CURDIR)/deployments/container/Dockerfile
 
 # Use a generic build target to build the relevant images
 $(IMAGE_TARGETS): image-%:
@@ -90,7 +87,6 @@ $(IMAGE_TARGETS): image-%:
 .PHONY: build
 build: $(DEFAULT_PUSH_TARGET)
 $(DEFAULT_PUSH_TARGET): build-$(DEFAULT_PUSH_TARGET)
-$(DEFAULT_PUSH_TARGET): DIST = $(DEFAULT_PUSH_TARGET)
 
 REGCTL ?= regctl
 $(PUSH_TARGETS): push-%:
diff --git a/deployments/container/cleanup/cleanup.sh b/deployments/container/cleanup/cleanup.sh
deleted file mode 100755
index 00c22da5d..000000000
--- a/deployments/container/cleanup/cleanup.sh
+++ /dev/null
@@ -1,78 +0,0 @@
-#!/bin/bash
-# Copyright 2024 NVIDIA CORPORATION
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express orimplied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -x
-
-rpm -qa | sort -u > package-list.original
-
-echo "install_weak_deps=False" >> /etc/dnf/dnf.conf
-rm -f /etc/dnf/protected.d/*.conf
-
-rm -f /etc/yum.repos.d/cuda.repo
-rm -f /etc/ld.so.conf.d/nvidia.conf
-
-dnf remove -y \
-    cuda* \
-    systemd
-
-# Remove the CUDA public key
-for key in $(rpm -qa gpg-pubkey*); do
-    rpm -qi ${key} | grep -o "cudatools <cudatools@nvidia.com>"
-    if [[ $? -eq 0 ]]; then
-        rpm -e ${key}
-    fi
-done
-
-dnf clean -y all
-rm -rf /var/cache/dnf
-
-dnf install -y microdnf
-
-microdnf remove -y $(rpm -q --whatrequires dnf)
-rpm -e dnf
-
-microdnf remove -y \
-    $(rpm -q --whatrequires /usr/libexec/platform-python) \
-    $(rpm -q --whatrequires 'python(abi)') \
-    python* \
-    dnf*
-
-microdnf remove -y \
-    $(rpm -qa | sort | grep -v -f package-names.minimal -e gpg-pubkey)
-
-# We need mount for our init container and install it here.
-# TODO: We need to provide a better way to install this / skip the cleanup of this package.
-microdnf install -y util-linux-core
-
-# We don't want to add third-party content to the base image and only remove packages.
-# We therefore skip running microdnf update here
-# microdnf update
-
-microdnf clean all
-rpm -e microdnf libdnf libpeas
-rm -rf /var/lib/dnf
-
-set +x
-rpm -qa | sort -u > package-list.cleaned
-for p in $(rpm -qa | sort -u); do
-    echo "START $p" >> package-list.cleaned.info
-    echo "INFO" >> package-list.cleaned.info
-    rpm -qi $p >> package-list.cleaned.info
-    echo "REQUIRES" >> package-list.cleaned.info
-    rpm -qR $p >> package-list.cleaned.info
-    echo "END $p" >>  package-list.cleaned.info
-done
-
-rm -rf /var/cache/dnf