Use pipx to install Python dependencies

recipes/linux/cleanup.sh

@@ -20,3 +20,4 @@ rm -rf /var/log/* || echo "'rm -rf /var/log/*' failed"
 rm -rf /root/.cache/* || echo "'rm -rf /root/.cache/*' failed"
 rm -rf /root/.cargo/* || echo "'rm -rf /root/.cargo/*' failed"
 rm -rf /tmp/* || echo "'rm -rf /tmp/*' failed"
+rm -rf /opt/pipx/logs/* || echo "'rm -rf /opt/pipx/logs/*' failed"

recipes/linux/fuzzfetch.sh

@@ -20,13 +20,9 @@ case "${1-install}" in
       lbzip2 \
       python3
     apt-install-auto \
-      gcc \
-      python3-dev \
-      python3-pip \
-      python3-setuptools \
-      python3-wheel
+      pipx
 
-    retry pip3 install fuzzfetch
+    PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin retry pipx install fuzzfetch
     ;;
   test)
     fuzzfetch -h

recipes/linux/fuzzing_tc.sh

@@ -14,25 +14,21 @@ source "${0%/*}/common.sh"
 case "${1-install}" in
   install)
     # assert that SRCDIR is set
-    [ -n "$SRCDIR" ]
+    [[ -n "$SRCDIR" ]]
 
     sys-embed \
       ca-certificates \
       git \
       openssh-client \
-      python3 \
-      python3-setuptools
+      python3
     apt-install-auto \
-      gcc \
-      python3-dev \
-      python3-pip \
-      python3-wheel
+      pipx
 
-    if [ "$EDIT" = "1" ]
+    if [[ "$EDIT" = "1" ]]
     then
-      retry pip3 install --no-build-isolation -e "$SRCDIR"
+      PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin retry pipx install -e "$SRCDIR"
     else
-      retry pip3 install "$SRCDIR"
+      PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin retry pipx install "$SRCDIR"
     fi
     ;;
   test)

recipes/linux/gsutil.sh

@@ -16,16 +16,11 @@ source "${0%/*}/common.sh"
 case "${1-install}" in
   install)
     sys-embed \
-      python3 \
-      python3-setuptools \
-      zstd
+      python3
     apt-install-auto \
-      gcc \
-      python3-dev \
-      python3-pip \
-      python3-wheel
+      pipx
 
-    retry pip3 install gsutil
+    PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin retry pipx install gsutil
     ;;
   test)
     gsutil version

recipes/linux/pernosco_submit.sh

@@ -21,26 +21,27 @@ case "${1-install}" in
       python3 \
       zstd
     apt-install-auto \
-      curl \
-      gcc \
       git \
-      python3-dev \
-      python3-pip \
-      python3-setuptools \
-      python3-wheel
-    retry pip3 install awscli
+      pipx \
+      python3-venv
+    PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin retry pipx install awscli
+    python3 -m venv /opt/venvs/pernosco
 
-    python_path="$(python3 -c 'import distutils.sysconfig;print(distutils.sysconfig.get_python_lib())')"
+    python_path="$(/opt/venvs/pernosco/bin/python3 -c 'import distutils.sysconfig;print(distutils.sysconfig.get_python_lib())')"
     TMPD="$(mktemp -d -p. pernosco.build.XXXXXXXXXX)"
     pushd "$TMPD" >/dev/null
       git-clone "https://github.com/pernosco/pernosco-submit"
       cp -r pernosco-submit/pernoscoshared "$python_path"
       cp pernosco-submit/pernosco-submit /usr/local/bin
     popd >/dev/null
     rm -rf "$TMPD"
+    sed -i '1 s,^.*$,#!/opt/venvs/pernosco/bin/python3,' /usr/local/bin/pernosco-submit
     chmod +x /usr/local/bin/pernosco-submit
     ;;
   test)
     pernosco-submit --help
+    aws --version
+    openssl version
+    zstdmt --version
     ;;
 esac

recipes/linux/prefpicker.sh

@@ -16,17 +16,12 @@ source "${0%/*}/common.sh"
 case "${1-install}" in
   install)
     sys-embed \
-      git \
       python3
     apt-install-auto \
-      ca-certificates \
-      python3-pip \
-      python3-setuptools \
-      python3-wheel
+      git \
+      pipx
 
-    git-clone "https://github.com/MozillaSecurity/prefpicker.git"
-    cd prefpicker
-    pip install .
+    PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin retry pipx install git+https://github.com/MozillaSecurity/prefpicker.git
     ;;
   test)
     prefpicker -h

services/libfuzzer/Dockerfile

@@ -6,26 +6,18 @@ FROM ubuntu:22.04
 
 LABEL maintainer Jesse Schwartzentruber <[email protected]>
 
-ENV LOGNAME         worker
-ENV HOSTNAME        taskcluster-worker
-ARG DEBIAN_FRONTEND=noninteractive
-
-RUN useradd -d /home/worker -s /bin/bash -m worker
-
 COPY recipes/linux/ /tmp/recipes/
 COPY services/libfuzzer/setup.sh /tmp/recipes/
 COPY services/fuzzing-decision /tmp/fuzzing-tc
 COPY base/linux/etc/pip.conf /etc/pip.conf
 RUN /tmp/recipes/setup.sh \
-    && rm -rf /tmp/recipes /tmp/fuzzing-tc
-
-COPY services/libfuzzer/launch.sh /home/worker/
-COPY services/libfuzzer/libfuzzer.sh /home/worker/
-COPY services/libfuzzer/coverage.sh /home/worker/
-COPY services/libfuzzer/setup-target.sh /home/worker/
+  && rm -rf /tmp/recipes /tmp/fuzzing-tc
 
-ENV LANG   en_US.UTF-8
-ENV LC_ALL en_US.UTF-8
+COPY services/libfuzzer/launch.sh \
+  services/libfuzzer/libfuzzer.sh \
+  services/libfuzzer/coverage.sh \
+  services/libfuzzer/setup-target.sh \
+  /home/worker/
 
 WORKDIR /home/worker
 ENTRYPOINT ["/usr/local/bin/fuzzing-pool-launch"]

services/libfuzzer/launch.sh

@@ -82,7 +82,7 @@ EOF
     cd /src/guided-fuzzing-daemon || exit 1
     retry git fetch origin main
     git reset --hard origin/main
-    pip3 install .
+    PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin retry pipx upgrade guided-fuzzing-daemon
   )
     then
     echo "Failed to install guided fuzzing daemon!"
@@ -128,4 +128,4 @@ then
   exit 0
 else
   exit $exit_code
-fi
+fi

services/libfuzzer/libfuzzer.sh

@@ -16,7 +16,7 @@ source ~/.local/bin/common.sh
 
 gcs-cat () {
 # gcs-cat bucket path
-python3 - "$1" "$2" << "EOF"
+/opt/pipx/venvs/guided-fuzzing-daemon/bin/python - "$1" "$2" << "EOF"
 import os
 import sys
 from google.cloud import storage

services/libfuzzer/setup.sh

@@ -7,6 +7,9 @@ set -e
 set -x
 set -o pipefail
 
+DEBIAN_FRONTEND=noninteractive
+export DEBIAN_FRONTEND
+
 # shellcheck source=recipes/linux/common.sh
 source "${0%/*}/common.sh"
 
@@ -24,14 +27,21 @@ SRCDIR=/tmp/fuzzing-tc ./fuzzing_tc.sh
 ./llvm-symbolizer.sh
 ./nodejs.sh
 ./taskcluster.sh
+./worker.sh
 
 packages=(
   apt-utils
   binutils
   bzip2
+  chromium-codecs-ffmpeg-extra
   curl
   git
   gpg-agent
+  gstreamer1.0-gl
+  gstreamer1.0-libav
+  gstreamer1.0-plugins-base
+  gstreamer1.0-plugins-ugly
+  gstreamer1.0-vaapi
   jshon
   lbzip2
   less
@@ -41,33 +51,25 @@ packages=(
   locales
   nano
   openssh-client
+  pipx
   psmisc
-  python3-pip
   ripgrep
   screen
   software-properties-common
+  subversion
+  ubuntu-restricted-addons
   unzip
+  wget  # used by oss-fuzz/infra/helper.py
   xvfb
   zip
 )
-package_recommends=(
-  subversion
-  ubuntu-restricted-addons
-  # used by oss-fuzz/infra/helper.py
-  wget
-)
 
 sys-embed "${packages[@]}"
-# want recommends for these packages
-retry apt-get install -y -qq "${package_recommends[@]}"
 apt-install-depends firefox
 apt-get remove --purge -qq xul-ext-ubufox
 
 #### Base System Configuration
 
-# Generate locales
-locale-gen en_US.utf8
-
 # Ensure the machine uses core dumps with PID in the filename
 # https://github.com/moby/moby/issues/11740
 cat << EOF | tee /etc/sysctl.d/60-fuzzos.conf > /dev/null
@@ -108,9 +110,7 @@ git remote add origin "https://github.com/MozillaSecurity/guided-fuzzing-daemon"
 retry git fetch origin main
 git checkout main
 cd -
-# install then uninstall so only dependencies remain
-retry pip3 install google-cloud-storage /src/guided-fuzzing-daemon
-pip3 uninstall -y guided-fuzzing-daemon
+PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin retry pipx install /src/guided-fuzzing-daemon
 
 /home/worker/.local/bin/cleanup.sh
 

services/test-recipes/Dockerfile-fuzzing_tc

@@ -2,7 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 
 LABEL maintainer Jesse Schwartzentruber <[email protected]>
 

services/test-recipes/Dockerfile-orion_decision

@@ -2,7 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 
 LABEL maintainer Jesse Schwartzentruber <[email protected]>