Skip to content

Commit 2de75fa

Browse files
ramperherramperher
committed
Test support for Mellanox interfaces
1 parent 322f394 commit 2de75fa

File tree

9 files changed

+1701
-16
lines changed

9 files changed

+1701
-16
lines changed

‎grout-container-app/cnfapp/Dockerfile

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
## Image to build webserver
2-
FROM docker.io/library/golang:1.23 as build
2+
FROM registry.access.redhat.com/ubi9/go-toolset:1.23 as build
33

4-
WORKDIR /utils
4+
WORKDIR /opt/app-root/src
55
COPY utils/webserver.go .
66
RUN go mod init webserver.go
77
RUN GOOS=linux CGO_ENABLED=0 go build -a -o webserver .
@@ -53,7 +53,7 @@ RUN chmod 750 /var/log/grout
5353
RUN chown example-cnf /var/log/grout
5454

5555
# Copy scripts
56-
COPY --chmod=550 --from=build /utils/webserver /usr/local/bin/webserver
56+
COPY --chmod=550 --from=build /opt/app-root/src/webserver /usr/local/bin/webserver
5757
COPY --chmod=550 scripts/grout-wrapper /usr/local/bin/example-cnf/grout-wrapper
5858
COPY --chmod=550 scripts/retrieve-grout-ip-addresses /usr/local/bin/example-cnf/retrieve-grout-ip-addresses
5959

‎testpmd-container-app/cnfapp/Dockerfile

+3-3
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@
22
FROM quay.io/rh-nfv-int/dpdk:v0.0.1 as build
33

44
## Image to build webserver
5-
FROM docker.io/library/golang:1.23 as build2
5+
FROM registry.access.redhat.com/ubi9/go-toolset:1.23 as build2
66

7-
WORKDIR /utils
7+
WORKDIR /opt/app-root/src
88
COPY utils/webserver.go .
99
RUN go mod init webserver.go
1010
RUN GOOS=linux CGO_ENABLED=0 go build -a -o webserver .
@@ -43,7 +43,7 @@ RUN chmod 750 /var/log/testpmd
4343
RUN chown example-cnf /var/log/testpmd
4444

4545
# Copy scripts
46-
COPY --chmod=550 --from=build2 /utils/webserver /usr/local/bin/webserver
46+
COPY --chmod=550 --from=build2 /opt/app-root/src/webserver /usr/local/bin/webserver
4747
COPY --chmod=550 --from=build /usr/local/bin/dpdk-testpmd /usr/local/bin/example-cnf/testpmd
4848
COPY --chmod=550 scripts/testpmd-wrapper /usr/local/bin/example-cnf/testpmd-wrapper
4949

‎testpmd-container-app/cnfapp/scripts/testpmd-wrapper

+6-2
Original file line numberDiff line numberDiff line change
@@ -45,13 +45,17 @@ if [ -f /sys/fs/cgroup/cpuset/cpuset.cpus ]; then
4545
elif [ -f /sys/fs/cgroup/cpuset.cpus ]; then
4646
# Applied to latest OCP 4.16 nightlies starting from 4.16.0 ec.5
4747
LCORES=$(cat /sys/fs/cgroup/cpuset.cpus)
48+
elif [ -f /proc/1/status ]; then
49+
# Applied when using a privileged pod, since /sys/fs/cgroup directory is overriden with host configuration
50+
# Extract the list of CPUs from /proc/1/status file
51+
LCORES=$(cat /proc/1/status | grep Cpus_allowed_list | awk '{print $2}')
4852
else
49-
echo "Could not find file to extract cores, existing.."
53+
echo "Could not find file to extract cores, exiting..."
5054
exit 1
5155
fi
5256

5357
if [ -z $LCORES ]; then
54-
echo "Could not find cores, existing.."
58+
echo "Could not find cores, exiting..."
5559
exit 1
5660
fi
5761

‎trex-container-app/app/Dockerfile

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
FROM docker.io/library/golang:1.23 as build
1+
FROM registry.access.redhat.com/ubi9/go-toolset:1.23 as build
22

3-
WORKDIR /utils
3+
WORKDIR /opt/app-root/src
44
COPY utils/webserver.go .
55
RUN go mod init webserver.go
66
RUN GOOS=linux CGO_ENABLED=0 go build -a -o webserver .
@@ -45,7 +45,7 @@ RUN chown example-cnf:example-cnf /var/log/trex
4545

4646
# Copy scripts
4747
COPY --chmod=550 scripts /usr/local/bin/
48-
COPY --chmod=550 --from=build /utils/webserver /usr/local/bin/webserver
48+
COPY --chmod=550 --from=build /opt/app-root/src/webserver /usr/local/bin/webserver
4949
COPY --chmod=550 pyfiles /opt/pyfiles/
5050

5151
# Move to the custom user

‎trex-container-app/server/Dockerfile

+6-5
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
1-
FROM docker.io/library/golang:1.23 as build
1+
FROM registry.access.redhat.com/ubi9/go-toolset:1.23 as build
22

3-
WORKDIR /utils
3+
WORKDIR /opt/app-root/src
44
COPY utils/webserver.go .
55
RUN go mod init webserver.go
66
RUN GOOS=linux CGO_ENABLED=0 go build -a -o webserver .
77

88
## trex image
9-
FROM quay.io/rh-nfv-int/ubi8-base-trex:v0.0.1
9+
FROM quay.io/rh-nfv-int/ubi9-base-trex-mellanox-support:v0.0.1
1010

1111
LABEL name="NFV Example TRexServer Application" \
1212
maintainer="telcoci" \
@@ -46,9 +46,10 @@ RUN dnf install -y \
4646
vim \
4747
&& dnf clean all
4848

49+
# Install TRex
4950
RUN mkdir -p /opt/trex && cd /opt/trex && git clone --branch ${TREX_VER} ${TREX_REPO}
5051
RUN cd /opt/trex/trex-core/linux_dpdk && \
51-
./b configure --no-mlx --no-bxnt --new-memory && \
52+
./b configure --no-bnxt --new-memory && \
5253
./b build
5354

5455
RUN cd /opt && git clone ${TRAFFICGEN_REPO} && mv bench-trafficgen/trafficgen . && rm -rf bench-trafficgen
@@ -79,7 +80,7 @@ RUN chmod 664 /usr/local/bin/example-cnf/trex_cfg.yaml
7980

8081
# Copy scripts
8182
COPY --chmod=550 scripts /usr/local/bin
82-
COPY --chmod=550 --from=build /utils/webserver /usr/local/bin/webserver
83+
COPY --chmod=550 --from=build /opt/app-root/src/webserver /usr/local/bin/webserver
8384

8485
# Move to the custom user
8586
USER example-cnf

‎utils/support-images/ubi9-base-trex-mellanox-support/Dockerfile

+145
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
# Common (multistage) args
2+
ARG D_OS="rhel9.2"
3+
ARG D_ARCH="x86_64"
4+
ARG D_CONTAINER_VER="0"
5+
ARG D_DOCA_VERSION="2.9.1"
6+
ARG D_OFED_VERSION="24.10-1.1.4.0"
7+
ARG D_KERNEL_VER="5.14.0-284.32.1.el9_2.x86_64"
8+
ARG D_OFED_SRC_DOWNLOAD_PATH="/run/mellanox/src"
9+
ARG OFED_SRC_LOCAL_DIR=${D_OFED_SRC_DOWNLOAD_PATH}/MLNX_OFED_SRC-${D_OFED_VERSION}
10+
11+
# Final clean image of precompiled driver container
12+
ARG D_FINAL_BASE_IMAGE=quay.io/rh-nfv-int/ubi8-base-trex:v0.0.1
13+
14+
##################################################################
15+
# Stage: Minimal base image update and install common requirements
16+
17+
# DTK base image (below example for specific kernel headers version)
18+
ARG D_BASE_IMAGE="registry.redhat.io/openshift4/driver-toolkit-rhel9:v4.13.0-202309112001.p0.gd719bdc.assembly.stream"
19+
# Standart: registry.access.redhat.com/ubi9:latest
20+
21+
ARG D_PYTHON_VERSION="36"
22+
ARG D_PYTHON="python${D_PYTHON_VERSION}"
23+
24+
FROM $D_BASE_IMAGE AS base
25+
26+
# Inherited global args
27+
ARG D_OS
28+
29+
# https://redmine.mellanox.com/issues/3528150/issue_history#note-9
30+
RUN if [[ "${D_OS}" == *"rhel9"* ]] ; then \
31+
sed -i 's#/etc/pki/entitlement#/etc/pki/entitlement-host#g' /etc/rhsm/rhsm.conf ;\
32+
fi
33+
34+
RUN set -x && \
35+
# Driver build / install script requirements
36+
dnf -y install perl \
37+
# Container functional requirements
38+
jq iproute kmod procps-ng udev
39+
40+
##############################################################################################
41+
# Stage: Download NVIDIA driver sources and install src driver container packages requirements
42+
43+
FROM base AS driver-src
44+
45+
# Inherited global args
46+
ARG D_DOCA_VERSION
47+
ARG D_OFED_VERSION
48+
ARG D_CONTAINER_VER
49+
ARG D_OFED_SRC_DOWNLOAD_PATH
50+
51+
# Stage args
52+
ARG D_OFED_BASE_URL="https://linux.mellanox.com/public/repo/doca/${D_DOCA_VERSION}/SOURCES/MLNX_OFED"
53+
ARG D_OFED_SRC_TYPE=""
54+
55+
ARG D_OFED_SRC_ARCHIVE="MLNX_OFED_SRC-${D_OFED_SRC_TYPE}${D_OFED_VERSION}.tgz"
56+
ARG D_OFED_URL_PATH="${D_OFED_BASE_URL}/${D_OFED_SRC_ARCHIVE}" # although argument name says URL, local `*.tgz` compressed files may also be used (intended for internal use)
57+
58+
ENV NVIDIA_NIC_DRIVER_VER=${D_OFED_VERSION}
59+
ENV NVIDIA_NIC_CONTAINER_VER=${D_CONTAINER_VER}
60+
ENV NVIDIA_NIC_DRIVER_PATH="${D_OFED_SRC_DOWNLOAD_PATH}/MLNX_OFED_SRC-${D_OFED_VERSION}"
61+
62+
WORKDIR /root
63+
RUN set -x && \
64+
# Install prerequirements
65+
dnf install -y curl --allowerasing \
66+
# Driver build requirements
67+
autoconf python3-devel ethtool automake pciutils libtool hostname dracut
68+
69+
# Download NVIDIA NIC driver
70+
RUN mkdir -p ${D_OFED_SRC_DOWNLOAD_PATH}
71+
WORKDIR ${D_OFED_SRC_DOWNLOAD_PATH}
72+
ADD ${D_OFED_URL_PATH} ${D_OFED_SRC_ARCHIVE}
73+
RUN if file ${D_OFED_SRC_ARCHIVE} | grep compressed; then \
74+
tar -xzf ${D_OFED_SRC_ARCHIVE}; \
75+
else \
76+
mv ${D_OFED_SRC_ARCHIVE}/MLNX_OFED_SRC-${D_OFED_VERSION} . ; \
77+
fi
78+
79+
WORKDIR /
80+
ADD ./entrypoint.sh /root/entrypoint.sh
81+
ADD ./dtk_nic_driver_build.sh /root/dtk_nic_driver_build.sh
82+
83+
ENTRYPOINT ["/root/entrypoint.sh"]
84+
CMD ["sources"]
85+
86+
LABEL doca-version=${D_DOCA_VERSION}
87+
LABEL ofed-version=${D_OFED_VERSION}
88+
89+
#####################
90+
# Stage: Build driver
91+
92+
FROM driver-src AS driver-builder
93+
94+
# Inherited global args
95+
ARG D_OS
96+
ARG D_KERNEL_VER
97+
ARG OFED_SRC_LOCAL_DIR
98+
99+
RUN set -x && \
100+
# MOFED installation requirements
101+
dnf install -y autoconf gcc make rpm-build
102+
103+
# Build driver
104+
RUN set -x && \
105+
${OFED_SRC_LOCAL_DIR}/install.pl --without-depcheck --distro ${D_OS} --kernel ${D_KERNEL_VER} --kernel-sources /lib/modules/${D_KERNEL_VER}/build --kernel-only --build-only --without-iser --without-srp --without-isert --without-knem --without-xpmem --with-mlnx-tools --with-ofed-scripts --copy-ifnames-udev
106+
107+
###################################
108+
# Stage: Install precompiled driver
109+
110+
ARG D_FINAL_BASE_IMAGE
111+
112+
FROM $D_FINAL_BASE_IMAGE AS precompiled
113+
114+
# Inherited global args
115+
ARG D_ARCH
116+
ARG D_KERNEL_VER
117+
ARG D_OFED_VERSION
118+
ARG D_CONTAINER_VER
119+
ARG OFED_SRC_LOCAL_DIR
120+
121+
ENV NVIDIA_NIC_DRIVER_VER=${D_OFED_VERSION}
122+
ENV NVIDIA_NIC_DRIVER_PATH=""
123+
ENV NVIDIA_NIC_CONTAINER_VER=${D_CONTAINER_VER}
124+
125+
COPY --from=driver-builder ${OFED_SRC_LOCAL_DIR}/RPMS/redhat-release-*/${D_ARCH}/*.rpm /root/
126+
127+
RUN rpm -ivh --nodeps /root/*.rpm
128+
129+
RUN set -x && \
130+
# MOFED functional requirements
131+
dnf install -y pciutils hostname udev ethtool \
132+
# Container functional requirements
133+
jq iproute kmod procps-ng udev
134+
135+
# Prevent modprobe from giving a WARNING about missing files
136+
RUN touch /lib/modules/${D_KERNEL_VER}/modules.order /lib/modules/${D_KERNEL_VER}/modules.builtin && \
137+
# Introduce installed kernel modules
138+
depmod ${D_KERNEL_VER}
139+
140+
WORKDIR /
141+
ADD ./entrypoint.sh /root/entrypoint.sh
142+
ADD ./dtk_nic_driver_build.sh /root/dtk_nic_driver_build.sh
143+
144+
ENTRYPOINT ["/root/entrypoint.sh"]
145+
CMD ["precompiled"]

‎utils/support-images/ubi9-base-trex-mellanox-support/dtk_nic_driver_build.sh

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
#!/bin/bash
2+
# Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
4+
: ${ENTRYPOINT_DEBUG:=false}
5+
: ${DTK_OCP_NIC_SHARED_DIR:=/mnt/shared-nvidia-nic-driver-toolkit}
6+
7+
DTK_OCP_START_COMPILE_FLAG=""
8+
DTK_OCP_DONE_COMPILE_FLAG=""
9+
DTK_OCP_COMPILED_DRIVER_VER=""
10+
11+
RETRY_DELAY_SEC=3
12+
13+
# Modified by main entrypoint prior to execution
14+
append_driver_build_flags=""
15+
16+
function timestamp_print () {
17+
date_time_stamp=$(date +'%d-%b-%y_%H:%M:%S')
18+
echo "[${date_time_stamp}] $@"
19+
}
20+
21+
function debug_print() {
22+
[ ${ENTRYPOINT_DEBUG} ] && timestamp_print $@
23+
}
24+
25+
# Function to execute command, capture exit status and stdout
26+
function exec_cmd() {
27+
debug_print "Executing command: $@"
28+
29+
output=$(eval "$@")
30+
exit_code=$?
31+
echo "$output"
32+
33+
if [[ $exit_code -ne 0 ]]; then
34+
echo "Command \"$@\" failed with exit code: $exit_code"
35+
exit $exit_code
36+
fi
37+
}
38+
39+
timestamp_print "DTK driver build script start"
40+
41+
if [ -z "${DTK_OCP_START_COMPILE_FLAG}" ] || [ -z "${DTK_OCP_DONE_COMPILE_FLAG}" ]; then
42+
timestamp_print "Compilation start/completion flags not set, aborting"
43+
exit 1
44+
fi
45+
46+
# Req. for /install.pl script
47+
exec_cmd "dnf install -y perl"
48+
# Req. for build
49+
exec_cmd "dnf install -y ethtool autoconf pciutils automake libtool python3-devel"
50+
51+
while [ ! -f ${DTK_OCP_START_COMPILE_FLAG} ]; do
52+
echo "Awaiting driver container preparations prior compilation, next query in ${RETRY_DELAY_SEC} sec"
53+
sleep ${RETRY_DELAY_SEC}
54+
done
55+
56+
timestamp_print "Starting compilation of driver version ${DTK_OCP_COMPILED_DRIVER_VER}"
57+
58+
exec_cmd "${DTK_OCP_NIC_SHARED_DIR}/MLNX_OFED_SRC-${DTK_OCP_COMPILED_DRIVER_VER}/install.pl --build-only --kernel-only --without-knem --without-iser --without-isert --without-srp --with-mlnx-tools --with-ofed-scripts --copy-ifnames-udev ${append_driver_build_flags}"
59+
60+
exec_cmd "touch ${DTK_OCP_DONE_COMPILE_FLAG}"
61+
62+
exec_cmd "rm ${DTK_OCP_START_COMPILE_FLAG}"
63+
64+
timestamp_print "DTK driver build script end"
65+
sleep infinity & wait

0 commit comments

Comments
 (0)