feat: update protobuf version (#172)

Changes:

  1. Update github actions base image to 22.04 since github is retiring 20.04.
  2. Fix docker container build and test.
  3. Update grpc tools to support protobuf 4.X.X.
  4. Remove pb2 files from git tracking.
  5. Bump DGP version to 2.0.0.

Author: kunlin596

.github/workflows/build-docker.yml

@@ -3,17 +3,17 @@ name: build-docker
 on:
   push:
     branches:
-      - 'master'
+      - "master"
     tags:
-      - '*'
+      - "*"
 
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
   build-and-push-image:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     permissions:
       contents: read
       packages: write

.github/workflows/coverage.yml

@@ -2,24 +2,24 @@ name: coverage
 on:
   pull_request:
     branches:
-      - 'master'
+      - "master"
   push:
     branches:
-      - 'master'
+      - "master"
 jobs:
   code-coverage:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       ENVIRONMENT: TEST_RUNNER
-      OS: ubuntu-20.04
-      PYTHON: '3.9'
+      OS: ubuntu-22.04
+      PYTHON: "3.9"
       COVERAGE_TOTAL: 49 # Coverage threshold percentage
     steps:
       - name: Checkout (admin token)
         if: ${{github.event_name != 'pull_request'}} # We don't want to use the admin token for PR flows
         uses: actions/checkout@master
         with:
-          token: '${{ secrets.GIT_ADMIN_WORKFLOW_TOKEN }}'
+          token: "${{ secrets.GIT_ADMIN_WORKFLOW_TOKEN }}"
           fetch-depth: "2" # Original commit + code cov badge commit
       - name: Checkout (normal flow)
         if: ${{github.event_name == 'pull_request'}}
@@ -34,17 +34,16 @@ jobs:
         id: coverage-installer
         run: |
           python -m pip install --upgrade pip
-          pip install cython==0.29.21 numpy==1.23.2
           sudo apt-get install jq
-          pip install -r requirements.txt
-          pip install -r requirements-dev.txt
           pip install coverage-badge
+          make develop
       - name: Run tests and calculate coverage
         id: test-runner
         run: |
           coverage run -m pytest
           coverage-badge -f -o docs/coverage.svg
-          COVERAGE_SCORE=$(coverage json -o /dev/stdout | jq .totals.percent_covered)
+          coverage json
+          COVERAGE_SCORE=$(jq '.totals.percent_covered' coverage.json)
           echo "::set-output name=coverageScore::$COVERAGE_SCORE"
       - name: Verify Changed files
         uses: tj-actions/verify-changed-files@v6

.github/workflows/doc-gen.yml

@@ -2,17 +2,17 @@ name: doc-gen
 on:
   push:
     branches:
-      - 'master'
+      - "master"
   pull_request:
     branches:
-      - 'master'
+      - "master"
 
 jobs:
   run:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
-      OS: ubuntu-20.04
-      PYTHON: '3.9'
+      OS: ubuntu-22.04
+      PYTHON: "3.9"
     steps:
       - uses: actions/checkout@master
         with:
@@ -26,7 +26,6 @@ jobs:
       - name: Setup requirements and run sphinx
         run: |
           python -m pip install --upgrade pip
-          pip install cython==0.29.21 numpy==1.23.2
           pip install -r requirements.txt
           pip install -r requirements-dev.txt
           pip install -r docs/requirements-doc.txt
@@ -39,4 +38,4 @@ jobs:
         with:
           branch: gh-pages
           folder: ./docs/build/html
-          commit-message: 'docs: update build documentation'
+          commit-message: "docs: update build documentation"

.github/workflows/pre-merge.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   pre-merge-tests:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       ENVIRONMENT: TEST_RUNNER
     steps:
@@ -28,7 +28,6 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install cython==0.29.21 numpy==1.23.2
           pip install -r requirements.txt
           pip install -r requirements-dev.txt
       - name: Lint all files with pre-commit

.gitignore

@@ -111,3 +111,13 @@ Pipfile
 Pipfile.lock
 
 \.vscode/
+*.ply
+
+*.pyi
+*_pb2.py
+*_pb2_grpc.py
+
+# Test data
+tests/data/dgp/autolabel_root/
+tests/data/dgp/test_scene/scene_01/autolabels/
+tests/data/dgp/test_scene/scene_02/autolabels/

.pre-commit-config.yaml

@@ -61,14 +61,21 @@ repos:
     hooks:
       - id: yapf
         # pre-commit requires that we split args at whitespce boundaries.
-        args: [
-          --style, .style.yapf,
-          -i,
-          -e, '*pb2.py',
-          -e, '*pb2_grpc.py',
-          -e, '*eggs*.py',
-          -e, '*env*',
-          -e, 'build/*'
-        ]
+        args:
+          [
+            --style,
+            .style.yapf,
+            -i,
+            -e,
+            "*pb2.py",
+            -e,
+            "*pb2_grpc.py",
+            -e,
+            "*eggs*.py",
+            -e,
+            "*env*",
+            -e,
+            "build/*",
+          ]
         exclude: '\w*pb2.py'
         additional_dependencies: [toml]

Dockerfile

@@ -1,8 +1,8 @@
-FROM nvidia/cuda:11.2.2-devel-ubuntu20.04
+FROM nvidia/cuda:11.6.1-cudnn8-devel-ubuntu20.04
 
 ARG python=3.9
-ENV PYTORCH_VERSION=1.8.1+cu111
-ENV TORCHVISION_VERSION=0.9.1+cu111
+ENV PYTORCH_VERSION=1.13.1+cu116
+ENV TORCHVISION_VERSION=0.14.1+cu116
 
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
@@ -42,11 +42,13 @@ RUN pip install --no-cache-dir \
   torchvision==${TORCHVISION_VERSION} \
   -f https://download.pytorch.org/whl/${PYTORCH_VERSION/*+/}/torch_stable.html
 
+
 # Install python dependencies
 ARG WORKSPACE=/home/dgp
 WORKDIR ${WORKSPACE}
 COPY requirements.txt requirements-dev.txt /tmp/
-RUN pip install --no-cache-dir cython==0.29.30 numpy==1.20.3
+# Install torch-compatible NumPy.
+RUN pip install numpy==1.26.4
 RUN pip install --no-cache-dir -r /tmp/requirements.txt
 RUN pip install --no-cache-dir -r /tmp/requirements-dev.txt
 

MANIFEST.in

@@ -1,3 +1,5 @@
 include README.md
+include requirements.txt
+include requirements-dev.txt
 include docs/*
 recursive-exclude * *.proto

Makefile

@@ -1,5 +1,5 @@
 # Copyright 2019-2021 Toyota Research Institute.  All rights reserved.
-PYTHON ?= python3
+PYTHON_EXEC ?= python3
 PACKAGE_NAME ?= dgp
 WORKSPACE ?= /home/$(PACKAGE_NAME)
 DOCKER_IMAGE_NAME ?= $(PACKAGE_NAME)
@@ -36,41 +36,42 @@ DOCKER_USER_OPTS ?= $(DOCKER_COMMON_OPTS) \
 UNITTEST ?= pytest
 UNITTEST_OPTS ?= -v
 
+.PHONY: clean build develop
+
 all: clean test
 
-build-proto:
+
+build:
 	PYTHONPATH=$(PWD):$(PYTHONPATH) \
-	$(PYTHON) setup.py build_py
+	DGP_DEV_VERSION=$(DEV_VERSION) $(PYTHON_EXEC) setup.py bdist_wheel
 
 clean:
-	$(PYTHON) setup.py clean && \
 	rm -rf build dist && \
 	find . -name "*.pyc" | xargs rm -f && \
 	find . -name "__pycache__" | xargs rm -rf
 	find . -name "*egg-info" | xargs rm -rf && \
 	find dgp/proto -name "*_grpc.py" | xargs rm -rf
 	find dgp/proto -name "*_pb2.py" | xargs rm -rf
 	find dgp/contribs/pd -name "*_pb2.py" | xargs rm -rf
+	find . -name "*eggs" | xargs rm -rf &
+	$(PYTHON_EXEC) setup.py clean
 
 develop:
-	pip install cython==0.29.30 numpy==1.20.3 grpcio==1.41.0 grpcio-tools==1.41.0
-	pip install --editable .
+	DGP_DEV_VERSION=$(DEV_VERSION) $(PYTHON_EXEC) -m pip install --editable ".[dev]"
 
 docker-build:
 	docker build \
 	--build-arg WORKSPACE=$(WORKSPACE) \
 	-t $(DOCKER_IMAGE) .
 
-docker-exec:
+docker-exec: docker-build
 	docker exec -it $(DOCKER_IMAGE_NAME) $(COMMAND)
 
-docker-run-tests: build-proto
-	docker run \
-	--name $(DOCKER_IMAGE_NAME)-tests \
-	$(DOCKER_ROOT_OPTS) $(DOCKER_IMAGE) \
-	$(UNITTEST) $(UNITTTEST_OPTS) $(WORKSPACE)/tests
+docker-run-tests: docker-build
+	docker run --name $(DOCKER_IMAGE_NAME)-tests $(DOCKER_ROOT_OPTS) $(DOCKER_IMAGE) make test
+
 
-docker-start-interactive:
+docker-start-interactive: docker-build
 	docker run \
 	$(DOCKER_USER_OPTS) \
 	$(DOCKER_IMAGE) \
@@ -83,9 +84,9 @@ setup-linters:
 	pre-commit install
 	pre-commit install --hook-type commit-msg
 
-test: build-proto
+test: develop
 	PYTHONPATH=$(PWD):$(PYTHONPATH) \
-	$(UNITTEST) $(UNITTEST_OPTS) $(PWD)/tests/
+	$(UNITTEST) $(UNITTEST_OPTS) $(PWD)/tests/ -vv
 
 unlink-githooks:
 	unlink .git/hooks/pre-push && unlink .git/hooks/pre-commit

README.md

@@ -62,11 +62,34 @@ You can build the base docker image and run the tests within
 [docker container](docs/GETTING_STARTED.md#markdown-header-develop-within-docker)
 via:
 
-```sh
+```shell
 make docker-build
 make docker-run-tests
 ```
 
+Build the Python wheel.
+
+```shell
+make build
+```
+
+For setup local developement.
+
+```shell
+make develop
+```
+
+Runing the test using local development environment.
+
+```shell
+make test
+```
+
+## Versioning
+
+This repository adheres to [PEP 440](https://peps.python.org/pep-0440/) for
+versioning.
+
 ## Contributing
 
 We appreciate all contributions to DGP! To learn more about making a

build_proto.py

@@ -0,0 +1,16 @@
+import os
+import subprocess
+from pathlib import Path
+
+_ROOT_DIRPATH = Path(__file__).parent.absolute()
+
+
+def build_proto(proto_version: str):
+    assert proto_version in {"proto3", "proto4"}, "Unsupported proto version."
+    from grpc_tools import command
+    command.build_package_protos(_ROOT_DIRPATH)
+
+
+if __name__ == "__main__":
+    proto_version = os.getenv("PROTO_VERSION", "proto3")
+    build_proto(proto_version)