style: use pre-commit to lint and format files (#3924)

Co-authored-by: Aaron Pham <[email protected]>

.bazelrc

@@ -1,3 +1,3 @@
-# load bazelrc from the legacy location as recommended 
+# load bazelrc from the legacy location as recommended
 # in https://github.com/bazelbuild/bazel/issues/6319
 import %workspace%/tools/bazel.rc

.devcontainer/Dockerfile

@@ -42,10 +42,10 @@ RUN --mount=type=cache,target=/var/lib/apt \
 COPY requirements/*.txt /tmp/pip-tmp/
 RUN --mount=type=cache,target=/root/.cache/pip \
     pip install --no-warn-script-location -r /tmp/pip-tmp/dev-requirements.txt -r /tmp/pip-tmp/docs-requirements.txt \
-    && rm -rf /tmp/pip-tmp
+    && rm -rf /tmp/pip-tmp && pre-commit install
 
-# Setting the ENTRYPOINT to docker-init.sh will configure non-root access to 
-# the Docker socket if "overrideCommand": false is set in devcontainer.json. 
+# Setting the ENTRYPOINT to docker-init.sh will configure non-root access to
+# the Docker socket if "overrideCommand": false is set in devcontainer.json.
 # The script will also execute CMD if you need to alter startup behaviors.
 ENTRYPOINT [ "/usr/local/share/docker-init.sh" ]
 CMD [ "sleep", "infinity" ]

.devcontainer/library-scripts/common-debian.sh

@@ -113,7 +113,7 @@ if [ "${PACKAGES_ALREADY_INSTALLED}" != "true" ]; then
         manpages \
         manpages-dev \
         init-system-helpers"
-        
+
     # Needed for adding manpages-posix and manpages-posix-dev which are non-free packages in Debian
     if [ "${ADD_NON_FREE_PACKAGES}" = "true" ]; then
         # Bring in variables from /etc/os-release like VERSION_CODENAME
@@ -124,7 +124,7 @@ if [ "${PACKAGES_ALREADY_INSTALLED}" != "true" ]; then
         sed -i -E "s/deb-src http:\/\/(deb|httpredir)\.debian\.org\/debian ${VERSION_CODENAME}-updates main/deb http:\/\/\1\.debian\.org\/debian ${VERSION_CODENAME}-updates main contrib non-free/" /etc/apt/sources.list
         sed -i "s/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main contrib non-free/" /etc/apt/sources.list
         sed -i "s/deb-src http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main contrib non-free/" /etc/apt/sources.list
-        sed -i "s/deb http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main/deb http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main contrib non-free/" /etc/apt/sources.list 
+        sed -i "s/deb http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main/deb http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main contrib non-free/" /etc/apt/sources.list
         sed -i "s/deb-src http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main/deb http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main contrib non-free/" /etc/apt/sources.list
         # Handle bullseye location for security https://www.debian.org/releases/bullseye/amd64/release-notes/ch-information.en.html
         sed -i "s/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}-security main/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}-security main contrib non-free/" /etc/apt/sources.list
@@ -140,7 +140,7 @@ if [ "${PACKAGES_ALREADY_INSTALLED}" != "true" ]; then
     if [[ ! -z $(apt-cache --names-only search ^libssl1.1$) ]]; then
         package_list="${package_list}       libssl1.1"
     fi
-    
+
     # Install appropriate version of libssl1.0.x if available
     libssl_package=$(dpkg-query -f '${db:Status-Abbrev}\t${binary:Package}\n' -W 'libssl1\.0\.?' 2>&1 || echo '')
     if [ "$(echo "$LIlibssl_packageBSSL" | grep -o 'libssl1\.0\.[0-9]:' | uniq | sort | wc -l)" -eq 0 ]; then
@@ -155,7 +155,7 @@ if [ "${PACKAGES_ALREADY_INSTALLED}" != "true" ]; then
 
     echo "Packages to verify are installed: ${package_list}"
     apt-get -y install --no-install-recommends ${package_list} 2> >( grep -v 'debconf: delaying package configuration, since apt-utils is not installed' >&2 )
-        
+
     # Install git if not already installed (may be more recent than distro version)
     if ! type git > /dev/null 2>&1; then
         apt-get -y install --no-install-recommends git
@@ -174,7 +174,7 @@ fi
 # Ensure at least the en_US.UTF-8 UTF-8 locale is available.
 # Common need for both applications and things like the agnoster ZSH theme.
 if [ "${LOCALE_ALREADY_SET}" != "true" ] && ! grep -o -E '^\s*en_US.UTF-8\s+UTF-8' /etc/locale.gen > /dev/null; then
-    echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen 
+    echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen
     locale-gen
     LOCALE_ALREADY_SET="true"
 fi
@@ -183,12 +183,12 @@ fi
 group_name="${USERNAME}"
 if id -u ${USERNAME} > /dev/null 2>&1; then
     # User exists, update if needed
-    if [ "${USER_GID}" != "automatic" ] && [ "$USER_GID" != "$(id -g $USERNAME)" ]; then 
+    if [ "${USER_GID}" != "automatic" ] && [ "$USER_GID" != "$(id -g $USERNAME)" ]; then
         group_name="$(id -gn $USERNAME)"
         groupmod --gid $USER_GID ${group_name}
         usermod --gid $USER_GID $USERNAME
     fi
-    if [ "${USER_UID}" != "automatic" ] && [ "$USER_UID" != "$(id -u $USERNAME)" ]; then 
+    if [ "${USER_UID}" != "automatic" ] && [ "$USER_UID" != "$(id -u $USERNAME)" ]; then
         usermod --uid $USER_UID $USERNAME
     fi
 else
@@ -198,7 +198,7 @@ else
     else
         groupadd --gid $USER_GID $USERNAME
     fi
-    if [ "${USER_UID}" = "automatic" ]; then 
+    if [ "${USER_UID}" = "automatic" ]; then
         useradd -s /bin/bash --gid $USERNAME -m $USERNAME
     else
         useradd -s /bin/bash --uid $USER_UID --gid $USERNAME -m $USERNAME
@@ -213,7 +213,7 @@ if [ "${USERNAME}" != "root" ] && [ "${EXISTING_NON_ROOT_USER}" != "${USERNAME}"
 fi
 
 # ** Shell customization section **
-if [ "${USERNAME}" = "root" ]; then 
+if [ "${USERNAME}" = "root" ]; then
     user_rc_path="/root"
 else
     user_rc_path="/home/${USERNAME}"
@@ -250,9 +250,9 @@ fi
 # Set the default git editor if not already set
 if [ -z "$(git config --get core.editor)" ] && [ -z "${GIT_EDITOR}" ]; then
     if  [ "${TERM_PROGRAM}" = "vscode" ]; then
-        if [[ -n $(command -v code-insiders) &&  -z $(command -v code) ]]; then 
+        if [[ -n $(command -v code-insiders) &&  -z $(command -v code) ]]; then
             export GIT_EDITOR="code-insiders --wait"
-        else 
+        else
             export GIT_EDITOR="code --wait"
         fi
     fi
@@ -329,7 +329,7 @@ codespaces_zsh="$(cat \
 # Codespaces zsh prompt theme
 __zsh_prompt() {
     local prompt_username
-    if [ ! -z "${GITHUB_USER}" ]; then 
+    if [ ! -z "${GITHUB_USER}" ]; then
         prompt_username="@${GITHUB_USER}"
     else
         prompt_username="%n"

.devcontainer/library-scripts/docker-debian.sh

@@ -86,7 +86,7 @@ find_version_from_git_tags() {
     local repository=$2
     local prefix=${3:-"tags/v"}
     local separator=${4:-"."}
-    local last_part_optional=${5:-"false"}    
+    local last_part_optional=${5:-"false"}
     if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then
         local escaped_separator=${separator//./\\.}
         local last_part
@@ -172,7 +172,7 @@ apt-get update
 if [ "${DOCKER_VERSION}" = "latest" ] || [ "${DOCKER_VERSION}" = "lts" ] || [ "${DOCKER_VERSION}" = "stable" ]; then
     # Empty, meaning grab whatever "latest" is in apt repo
     cli_version_suffix=""
-else    
+else
     # Fetch a valid version from the apt-cache (eg: the Microsoft repo appends +azure, breakfix, etc...)
     docker_version_dot_escaped="${DOCKER_VERSION//./\\.}"
     docker_version_dot_plus_escaped="${docker_version_dot_escaped//+/\\+}"
@@ -228,7 +228,7 @@ else
         fi
         ${pipx_bin} install --pip-args '--no-cache-dir --force-reinstall' docker-compose
         rm -rf /tmp/pip-tmp
-    else 
+    else
         compose_v1_version="1"
         find_version_from_git_tags compose_v1_version "https://github.com/docker/compose" "tags/"
         echo "(*) Installing docker-compose ${compose_v1_version}..."
@@ -286,13 +286,13 @@ usermod -aG docker "${USERNAME}"
 DOCKER_GID="$(grep -oP '^docker:x:\K[^:]+' /etc/group)"
 
 # If enabling non-root access and specified user is found, setup socat and add script
-chown -h "${USERNAME}":root "${TARGET_SOCKET}"        
+chown -h "${USERNAME}":root "${TARGET_SOCKET}"
 if ! dpkg -s socat > /dev/null 2>&1; then
     apt_get_update_if_needed
     apt-get -y install socat
 fi
 tee /usr/local/share/docker-init.sh > /dev/null \
-<< EOF 
+<< EOF
 #!/usr/bin/env bash
 #-------------------------------------------------------------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
@@ -324,8 +324,8 @@ log()
 echo -e "\n** \$(date) **" | sudoIf tee -a \${SOCAT_LOG} > /dev/null
 log "Ensuring ${USERNAME} has access to ${SOURCE_SOCKET} via ${TARGET_SOCKET}"
 
-# If enabled, try to update the docker group with the right GID. If the group is root, 
-# fall back on using socat to forward the docker socket to another unix socket so 
+# If enabled, try to update the docker group with the right GID. If the group is root,
+# fall back on using socat to forward the docker socket to another unix socket so
 # that we can set permissions on it without affecting the host.
 if [ "${ENABLE_NONROOT_DOCKER}" = "true" ] && [ "${SOURCE_SOCKET}" != "${TARGET_SOCKET}" ] && [ "${USERNAME}" != "root" ] && [ "${USERNAME}" != "0" ]; then
     SOCKET_GID=\$(stat -c '%g' ${SOURCE_SOCKET})
@@ -345,7 +345,7 @@ if [ "${ENABLE_NONROOT_DOCKER}" = "true" ] && [ "${SOURCE_SOCKET}" != "${TARGET_
     log "Success"
 fi
 
-# Execute whatever commands were passed in (if any). This allows us 
+# Execute whatever commands were passed in (if any). This allows us
 # to set this script to ENTRYPOINT while still executing the default CMD.
 set +e
 exec "\$@"

.github/ISSUE_TEMPLATE/config.yml

@@ -15,4 +15,4 @@ contact_links:
     about: Contact the BentoML team for support.
   - name: BentoML Blog
     url: modelserving.com
-    about: Read the latest blogs/updates from community and the BentoML team. 
+    about: Read the latest blogs/updates from community and the BentoML team.

.github/PULL_REQUEST_TEMPLATE.md

@@ -37,7 +37,7 @@ Fixes #(issue)
 
 - [ ] Does the Pull Request follow [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/#summary) naming? Here are [GitHub's
       guide](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request) on how to create a pull request.
-- [ ] Does the code follow BentoML's code style, both `make format` and `make lint` script have passed ([instructions](https://github.com/bentoml/BentoML/blob/main/DEVELOPMENT.md#style-check-auto-formatting-type-checking))?
+- [ ] Does the code follow BentoML's code style, `pre-commit run -a` script has passed ([instructions](https://github.com/bentoml/BentoML/blob/main/DEVELOPMENT.md#style-check-auto-formatting-type-checking))?
 - [ ] Did you read through [contribution guidelines](https://github.com/bentoml/BentoML/blob/main/CONTRIBUTING.md#ways-to-contribute) and follow [development guidelines](https://github.com/bentoml/BentoML/blob/main/DEVELOPMENT.md#start-developing)?
 - [ ] Did your changes require updates to the documentation? Have you updated
       those accordingly? Here are [documentation guidelines](https://github.com/bentoml/BentoML/tree/main/docs) and [tips on writting docs](https://github.com/bentoml/BentoML/tree/main/docs#writing-documentation).

.github/workflows/ci.yml

@@ -85,14 +85,9 @@ jobs:
       - name: Install dependencies
         run: |
           pip install .
-          pip install -r requirements/dev-requirements.txt
-      - name: Format check
-        run: |
-          black --check src examples tests
-          black --check --pyi typings
-          isort --check .
-      - name: Lint check
-        run: ruff check src tests examples
+          pip install pre-commit
+      - name: Format and lint check
+        run: pre-commit run --all-files
       - name: Type check
         if: ${{ github.event_name == 'pull_request' }}
         run: git diff --name-only --diff-filter=AM "origin/$GITHUB_BASE_REF" -z -- '*.py{,i}' | xargs -0 --no-run-if-empty pyright

.pre-commit-config.yaml

@@ -0,0 +1,25 @@
+ci:
+  autoupdate_schedule: monthly
+
+exclude: '.*\.(css|js|svg)$'
+repos:
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: 'v0.0.263'
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix, --show-fixes]
+
+  - repo: https://github.com/psf/black
+    rev: 23.3.0
+    hooks:
+      - id: black
+        additional_dependencies:  # black[jupyter] dependencies
+          - ipython>=7.8.0
+          - tokenize-rt>=3.2.0
+        files: '/(src|tests|docs|examples|typings)/'
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer

CONTRIBUTING.md

@@ -12,12 +12,12 @@ To follow development updates and discussion, join the #bentoml-contributors cha
 There are many ways to contribute to BentoML.
 
 * Supporting new users by answering questions on the
-    [github issues tracker](https://github.com/bentoml/BentoML/issues) and the 
+    [github issues tracker](https://github.com/bentoml/BentoML/issues) and the
     [#bentoml-users slack channel](https://join.slack.bentoml.org).
- 
-* Report issues you're facing and "Thumbs up" on issues and feature requests that are 
+
+* Report issues you're facing and "Thumbs up" on issues and feature requests that are
     relevant to you in BentoML's [issues tracker](https://github.com/bentoml/BentoML/issues).
- 
+
 * Investigate bugs and reviewing other developer's pull requests.
 
 * Contributing code or documentation to the project by submitting a Github pull request.
@@ -31,8 +31,8 @@ There are many ways to contribute to BentoML.
 We use Github issues to track all bugs and feature requests. Feel free to open an issue
 if you have found a bug or wish to see a new feature implemented.
 
-Before submitting a github issue, ensure the bug was not already reported under 
-[issues](https://github.com/bentoml/bentoml/issues) or currently being addressed by 
+Before submitting a github issue, ensure the bug was not already reported under
+[issues](https://github.com/bentoml/bentoml/issues) or currently being addressed by
 other [pull requests](https://github.com/bentoml/BentoML/pulls).
 
 If you're unable to find an open issue addressing the problem,
@@ -44,10 +44,10 @@ the expected behavior that is not occurring.
 
 ## Contributing Code
 
-To avoid duplicating work, it is highly recommended to search through the 
-[issue tracker](https://github.com/bentoml/bentoml/issues) and 
+To avoid duplicating work, it is highly recommended to search through the
+[issue tracker](https://github.com/bentoml/bentoml/issues) and
 [pull requests list](https://github.com/bentoml/BentoML/pulls). If in doubt about
-duplicated work, or if you want to work on a non-trivial feature, it's recommended to 
+duplicated work, or if you want to work on a non-trivial feature, it's recommended to
 first open an issue in the [issue tracker](https://github.com/bentoml/bentoml/issues)
 to get some feedbacks from core developers.
 
@@ -66,11 +66,11 @@ prior knoledge and help you get familiar with its codebase.
 ## Documentation
 
 Improving the documentation is no less important than improving the library. If you find
-a typo in the documentation, or have made improvements, do not hesitate to submit a 
-GitHub pull request. 
+a typo in the documentation, or have made improvements, do not hesitate to submit a
+GitHub pull request.
 
 Full documentation can be found under the `docs/source` directory. You can edit the
-documentation `.rst` or `.md` files using any text editor. Follow the instructions 
+documentation `.rst` or `.md` files using any text editor. Follow the instructions
 [here](https://github.com/bentoml/BentoML/blob/main/DEVELOPMENT.md#how-to-edit-run-build-documentation-site)
 to build documentation site locally, generate HTML output and preview your changes.
 
@@ -108,9 +108,9 @@ Tags for managing issues:
 High quality testing is extremely important for BentoML project. Currently BentoML has
 three kind of tests: Unit tests(`tests/`) and integrations (`tests/integration/`) are
 running on Travis CI for every pull request.  End-to-end tests(`e2e_tests/`) is manually
-executed by the maintainer before every release and for pull requests that are 
+executed by the maintainer before every release and for pull requests that are
 introducing major changes.
 
-We expect pull requests that are introducing new features to have at least 90% test 
+We expect pull requests that are introducing new features to have at least 90% test
 coverages. Pull requests that are fixing a bug should add a test covering the issue
 being fixed if possible.

DEVELOPMENT.md

@@ -209,23 +209,21 @@ bentoml get IrisClassifier --verbose
 
 ## Style check, auto-formatting, type-checking
 
-formatter: [black](https://github.com/psf/black), [isort](https://github.com/PyCQA/isort), [buf](https://github.com/bufbuild/buf)
+formatter: [black](https://github.com/psf/black), [buf](https://github.com/bufbuild/buf)
 
 linter: [ruff](https://github.com/charliermarsh/ruff), [buf](https://github.com/bufbuild/buf)
 
 type checker: [pyright](https://github.com/microsoft/pyright)
 
-We are using [buf](https://github.com/bufbuild/buf) for formatting and linting
+We are using [pre-commit](https://pre-commit.com/) to manage our hooks, and [buf](https://github.com/bufbuild/buf) for formatting and linting
 of our proto files. Configuration can be found [here](./bentoml/grpc/buf.yaml).
 Currently, we are running `buf` with docker, hence we kindly ask our developers
 to have docker available. Docker installation can be found [here](https://docs.docker.com/get-docker/).
 
 Run linter/format script:
 
 ```bash
-make format
-
-make lint
+pre-commit run --all-files
 ```
 
 Run type checker: