feat: add support for protobuf 4, update packaging and CI

Changes:

  1. Update github actions base image to 22.04 since github is retiring 20.04.
  2. Add DGP_PROTO_VERISON for controlling proto version.
  3. Remove previously generated old pb2 files from tracking.
  4. Fix docker container build and test.

Author: kunlin596

.github/workflows/build-docker.yml

@@ -3,17 +3,17 @@ name: build-docker
 on:
   push:
     branches:
-      - 'master'
+      - "master"
     tags:
-      - '*'
+      - "*"
 
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
   build-and-push-image:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     permissions:
       contents: read
       packages: write

.github/workflows/coverage.yml

@@ -2,24 +2,27 @@ name: coverage
 on:
   pull_request:
     branches:
-      - 'master'
+      - "master"
   push:
     branches:
-      - 'master'
+      - "master"
 jobs:
   code-coverage:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       ENVIRONMENT: TEST_RUNNER
-      OS: ubuntu-20.04
-      PYTHON: '3.9'
+      OS: ubuntu-22.04
+      PYTHON: "3.9"
       COVERAGE_TOTAL: 49 # Coverage threshold percentage
+    strategy:
+      matrix:
+        protobuf_major_version: [3, 4]
     steps:
       - name: Checkout (admin token)
         if: ${{github.event_name != 'pull_request'}} # We don't want to use the admin token for PR flows
         uses: actions/checkout@master
         with:
-          token: '${{ secrets.GIT_ADMIN_WORKFLOW_TOKEN }}'
+          token: "${{ secrets.GIT_ADMIN_WORKFLOW_TOKEN }}"
           fetch-depth: "2" # Original commit + code cov badge commit
       - name: Checkout (normal flow)
         if: ${{github.event_name == 'pull_request'}}
@@ -34,17 +37,19 @@ jobs:
         id: coverage-installer
         run: |
           python -m pip install --upgrade pip
-          pip install cython==0.29.21 numpy==1.23.2
           sudo apt-get install jq
           pip install -r requirements.txt
           pip install -r requirements-dev.txt
           pip install coverage-badge
+          DGP_PROTO_VERSION=3 pip install ".[dev]"
+          make develop PROTO_VERSION=${{ matrix.protobuf_major_version }}
       - name: Run tests and calculate coverage
         id: test-runner
         run: |
           coverage run -m pytest
           coverage-badge -f -o docs/coverage.svg
-          COVERAGE_SCORE=$(coverage json -o /dev/stdout | jq .totals.percent_covered)
+          coverage json
+          COVERAGE_SCORE=$(jq '.totals.percent_covered' coverage.json)
           echo "::set-output name=coverageScore::$COVERAGE_SCORE"
       - name: Verify Changed files
         uses: tj-actions/verify-changed-files@v6

.github/workflows/doc-gen.yml

@@ -2,17 +2,17 @@ name: doc-gen
 on:
   push:
     branches:
-      - 'master'
+      - "master"
   pull_request:
     branches:
-      - 'master'
+      - "master"
 
 jobs:
   run:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
-      OS: ubuntu-20.04
-      PYTHON: '3.9'
+      OS: ubuntu-22.04
+      PYTHON: "3.9"
     steps:
       - uses: actions/checkout@master
         with:
@@ -26,7 +26,6 @@ jobs:
       - name: Setup requirements and run sphinx
         run: |
           python -m pip install --upgrade pip
-          pip install cython==0.29.21 numpy==1.23.2
           pip install -r requirements.txt
           pip install -r requirements-dev.txt
           pip install -r docs/requirements-doc.txt
@@ -39,4 +38,4 @@ jobs:
         with:
           branch: gh-pages
           folder: ./docs/build/html
-          commit-message: 'docs: update build documentation'
+          commit-message: "docs: update build documentation"

.github/workflows/pre-merge.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   pre-merge-tests:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       ENVIRONMENT: TEST_RUNNER
     steps:
@@ -28,7 +28,6 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install cython==0.29.21 numpy==1.23.2
           pip install -r requirements.txt
           pip install -r requirements-dev.txt
       - name: Lint all files with pre-commit

.gitignore

@@ -111,3 +111,13 @@ Pipfile
 Pipfile.lock
 
 \.vscode/
+*.ply
+
+*.pyi
+*_pb2.py
+*_pb2_grpc.py
+
+# Test data
+tests/data/dgp/autolabel_root/
+tests/data/dgp/test_scene/scene_01/autolabels/
+tests/data/dgp/test_scene/scene_02/autolabels/

.pre-commit-config.yaml

@@ -61,14 +61,21 @@ repos:
     hooks:
       - id: yapf
         # pre-commit requires that we split args at whitespce boundaries.
-        args: [
-          --style, .style.yapf,
-          -i,
-          -e, '*pb2.py',
-          -e, '*pb2_grpc.py',
-          -e, '*eggs*.py',
-          -e, '*env*',
-          -e, 'build/*'
-        ]
+        args:
+          [
+            --style,
+            .style.yapf,
+            -i,
+            -e,
+            "*pb2.py",
+            -e,
+            "*pb2_grpc.py",
+            -e,
+            "*eggs*.py",
+            -e,
+            "*env*",
+            -e,
+            "build/*",
+          ]
         exclude: '\w*pb2.py'
         additional_dependencies: [toml]

Dockerfile

@@ -1,8 +1,8 @@
-FROM nvidia/cuda:11.2.2-devel-ubuntu20.04
+FROM nvidia/cuda:11.6.1-cudnn8-devel-ubuntu20.04
 
 ARG python=3.9
-ENV PYTORCH_VERSION=1.8.1+cu111
-ENV TORCHVISION_VERSION=0.9.1+cu111
+ENV PYTORCH_VERSION=1.13.1+cu116
+ENV TORCHVISION_VERSION=0.14.1+cu116
 
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
@@ -42,11 +42,13 @@ RUN pip install --no-cache-dir \
   torchvision==${TORCHVISION_VERSION} \
   -f https://download.pytorch.org/whl/${PYTORCH_VERSION/*+/}/torch_stable.html
 
+
 # Install python dependencies
 ARG WORKSPACE=/home/dgp
 WORKDIR ${WORKSPACE}
 COPY requirements.txt requirements-dev.txt /tmp/
-RUN pip install --no-cache-dir cython==0.29.30 numpy==1.20.3
+# Install torch-compatible NumPy.
+RUN pip install numpy==1.26.4
 RUN pip install --no-cache-dir -r /tmp/requirements.txt
 RUN pip install --no-cache-dir -r /tmp/requirements-dev.txt
 

Makefile

@@ -36,11 +36,18 @@ DOCKER_USER_OPTS ?= $(DOCKER_COMMON_OPTS) \
 UNITTEST ?= pytest
 UNITTEST_OPTS ?= -v
 
+.PHONY: clean build develop
+
 all: clean test
 
-build-proto:
+
+build:
+ifndef PROTO_VERSION
+	$(error PROTO_VERSION is not set. Please invoke make with PROTO_VERSION=X! X could be 3 or 4 now.)
+endif
+	@echo "Building the Python wheel for $(PROTO_VERSION)."
 	PYTHONPATH=$(PWD):$(PYTHONPATH) \
-	$(PYTHON) setup.py build_py
+	DGP_PROTO_VERSION=$(PROTO_VERSION) $(PYTHON) setup.py bdist_wheel
 
 clean:
 	$(PYTHON) setup.py clean && \
@@ -51,26 +58,31 @@ clean:
 	find dgp/proto -name "*_grpc.py" | xargs rm -rf
 	find dgp/proto -name "*_pb2.py" | xargs rm -rf
 	find dgp/contribs/pd -name "*_pb2.py" | xargs rm -rf
+	find . -name "*eggs" | xargs rm -rf
 
 develop:
-	pip install cython==0.29.30 numpy==1.20.3 grpcio==1.41.0 grpcio-tools==1.41.0
-	pip install --editable .
+ifndef PROTO_VERSION
+	$(error PROTO_VERSION is not set. Please invoke make with PROTO_VERSION=X! X could be 3 or 4 now.)
+endif
+	@echo "Install develop version for protobuf version $(PROTO_VERSION).X.X."
+	DGP_PROTO_VERSION=$(PROTO_VERSION) pip install --editable .
 
 docker-build:
 	docker build \
 	--build-arg WORKSPACE=$(WORKSPACE) \
 	-t $(DOCKER_IMAGE) .
 
-docker-exec:
+docker-exec: docker-build
 	docker exec -it $(DOCKER_IMAGE_NAME) $(COMMAND)
 
-docker-run-tests: build-proto
-	docker run \
-	--name $(DOCKER_IMAGE_NAME)-tests \
-	$(DOCKER_ROOT_OPTS) $(DOCKER_IMAGE) \
-	$(UNITTEST) $(UNITTTEST_OPTS) $(WORKSPACE)/tests
+docker-run-tests: docker-build
+ifndef PROTO_VERSION
+	$(error PROTO_VERSION is not set. Please invoke make with PROTO_VERSION=X! X could be 3 or 4 now.)
+endif
+	docker run --name $(DOCKER_IMAGE_NAME)-tests $(DOCKER_ROOT_OPTS) $(DOCKER_IMAGE) make PROTO_VERSION=$(PROTO_VERSION) test
+
 
-docker-start-interactive:
+docker-start-interactive: docker-build
 	docker run \
 	$(DOCKER_USER_OPTS) \
 	$(DOCKER_IMAGE) \
@@ -83,9 +95,9 @@ setup-linters:
 	pre-commit install
 	pre-commit install --hook-type commit-msg
 
-test: build-proto
+test: develop
 	PYTHONPATH=$(PWD):$(PYTHONPATH) \
-	$(UNITTEST) $(UNITTEST_OPTS) $(PWD)/tests/
+	$(UNITTEST) $(UNITTEST_OPTS) $(PWD)/tests/ -vv
 
 unlink-githooks:
 	unlink .git/hooks/pre-push && unlink .git/hooks/pre-commit

README.md

@@ -62,11 +62,46 @@ You can build the base docker image and run the tests within
 [docker container](docs/GETTING_STARTED.md#markdown-header-develop-within-docker)
 via:
 
-```sh
+`X` of 3 and 4 are supported.
+
+```shell
 make docker-build
-make docker-run-tests
+make PROTO_VERSION=X docker-run-tests
+```
+
+Build the Python wheel for vendoring different Protobuf versions.
+
+```shell
+make PROTO_VERSION=X build
 ```
 
+For setup local developement.
+
+```shell
+make PROTO_VERSION=X develop
+```
+
+Runing the test using local development environment.
+
+```shell
+make PROTO_VERSION=X test
+```
+
+## Versioning
+
+This repository adheres to [PEP 440](https://peps.python.org/pep-0440/) for
+versioning, with a slight modification:
+
+In addition to the standard `major.minor.micro` format, we append a fourth
+numeric segment to denote the `protoc` version used for compiling `.proto`
+files.
+
+For example, version `1.0.0.3` indicates that the package version is `1.0.0` and
+it was compiled using `protoc` version 3.x.x.
+
+This approach maintains compatibility with PEP 440 and allows users to select
+the appropriate package version corresponding to their `protoc` version.
+
 ## Contributing
 
 We appreciate all contributions to DGP! To learn more about making a

build_proto.py

@@ -0,0 +1,16 @@
+import os
+import subprocess
+from pathlib import Path
+
+_ROOT_DIRPATH = Path(__file__).parent.absolute()
+
+
+def build_proto(proto_version: str):
+    assert proto_version in {"proto3", "proto4"}, "Unsupported proto version."
+    from grpc_tools import command
+    command.build_package_protos(_ROOT_DIRPATH)
+
+
+if __name__ == "__main__":
+    proto_version = os.getenv("PROTO_VERSION", "proto3")
+    build_proto(proto_version)