feat: update all dependencies and project configs

Changes:

  1. Update github actions base image to 22.04 since github is retiring 20.04.
  2. Use pip-tools to compile all Python requirements.
  3. Update Python dependeny versions.
  4. Regenerated all pb2 files for proto3.
  5. Add extras_require for proto4.

Author: kunlin596

.github/workflows/build-docker.yml

@@ -3,17 +3,17 @@ name: build-docker
 on:
   push:
     branches:
-      - 'master'
+      - "master"
     tags:
-      - '*'
+      - "*"
 
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
   build-and-push-image:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     permissions:
       contents: read
       packages: write

.github/workflows/coverage.yml

@@ -2,24 +2,24 @@ name: coverage
 on:
   pull_request:
     branches:
-      - 'master'
+      - "master"
   push:
     branches:
-      - 'master'
+      - "master"
 jobs:
   code-coverage:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       ENVIRONMENT: TEST_RUNNER
-      OS: ubuntu-20.04
-      PYTHON: '3.9'
+      OS: ubuntu-22.04
+      PYTHON: "3.9"
       COVERAGE_TOTAL: 49 # Coverage threshold percentage
     steps:
       - name: Checkout (admin token)
         if: ${{github.event_name != 'pull_request'}} # We don't want to use the admin token for PR flows
         uses: actions/checkout@master
         with:
-          token: '${{ secrets.GIT_ADMIN_WORKFLOW_TOKEN }}'
+          token: "${{ secrets.GIT_ADMIN_WORKFLOW_TOKEN }}"
           fetch-depth: "2" # Original commit + code cov badge commit
       - name: Checkout (normal flow)
         if: ${{github.event_name == 'pull_request'}}
@@ -34,9 +34,8 @@ jobs:
         id: coverage-installer
         run: |
           python -m pip install --upgrade pip
-          pip install cython==0.29.21 numpy==1.23.2
           sudo apt-get install jq
-          pip install -r requirements.txt
+          pip install -r requirements_proto3.txt
           pip install -r requirements-dev.txt
           pip install coverage-badge
       - name: Run tests and calculate coverage

.github/workflows/doc-gen.yml

@@ -2,17 +2,17 @@ name: doc-gen
 on:
   push:
     branches:
-      - 'master'
+      - "master"
   pull_request:
     branches:
-      - 'master'
+      - "master"
 
 jobs:
   run:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
-      OS: ubuntu-20.04
-      PYTHON: '3.9'
+      OS: ubuntu-22.04
+      PYTHON: "3.9"
     steps:
       - uses: actions/checkout@master
         with:
@@ -26,8 +26,7 @@ jobs:
       - name: Setup requirements and run sphinx
         run: |
           python -m pip install --upgrade pip
-          pip install cython==0.29.21 numpy==1.23.2
-          pip install -r requirements.txt
+          pip install -r requirements_proto3.txt
           pip install -r requirements-dev.txt
           pip install -r docs/requirements-doc.txt
           cd docs
@@ -39,4 +38,4 @@ jobs:
         with:
           branch: gh-pages
           folder: ./docs/build/html
-          commit-message: 'docs: update build documentation'
+          commit-message: "docs: update build documentation"

.github/workflows/pre-merge.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   pre-merge-tests:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       ENVIRONMENT: TEST_RUNNER
     steps:
@@ -28,8 +28,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install cython==0.29.21 numpy==1.23.2
-          pip install -r requirements.txt
+          pip install -r requirements_proto3.txt
           pip install -r requirements-dev.txt
       - name: Lint all files with pre-commit
         run: |

.gitignore

@@ -111,3 +111,5 @@ Pipfile
 Pipfile.lock
 
 \.vscode/
+*.pyi
+*.ply

.pre-commit-config.yaml

@@ -61,14 +61,21 @@ repos:
     hooks:
       - id: yapf
         # pre-commit requires that we split args at whitespce boundaries.
-        args: [
-          --style, .style.yapf,
-          -i,
-          -e, '*pb2.py',
-          -e, '*pb2_grpc.py',
-          -e, '*eggs*.py',
-          -e, '*env*',
-          -e, 'build/*'
-        ]
+        args:
+          [
+            --style,
+            .style.yapf,
+            -i,
+            -e,
+            "*pb2.py",
+            -e,
+            "*pb2_grpc.py",
+            -e,
+            "*eggs*.py",
+            -e,
+            "*env*",
+            -e,
+            "build/*",
+          ]
         exclude: '\w*pb2.py'
         additional_dependencies: [toml]

dgp/__init__.py

@@ -1,52 +1,51 @@
 # Copyright 2021-2022 Toyota Research Institute. All rights reserved.
 import os
-from collections import OrderedDict
 
-__version__ = '1.0'
+__version__ = "1.6.0"
 
-DGP_PATH = os.getenv('DGP_PATH', default=os.getenv('HOME', os.getcwd()))
-DGP_DATA_DIR = os.path.join(DGP_PATH, '.dgp')
-DGP_CACHE_DIR = os.path.join(DGP_DATA_DIR, 'cache')
-DGP_DATASETS_CACHE_DIR = os.path.join(DGP_DATA_DIR, 'datasets')
+DGP_PATH = os.getenv("DGP_PATH", default=os.getenv("HOME", os.getcwd()))
+DGP_DATA_DIR = os.path.join(DGP_PATH, ".dgp")
+DGP_CACHE_DIR = os.path.join(DGP_DATA_DIR, "cache")
+DGP_DATASETS_CACHE_DIR = os.path.join(DGP_DATA_DIR, "datasets")
 
 TRI_DGP_FOLDER_PREFIX = "dgp/"
 TRI_RAW_FOLDER_PREFIX = "raw/"
 TRI_DGP_JSON_PREFIX = "dataset_v"
 
 # DGP Directory structure constants
-RGB_FOLDER = 'rgb'
-POINT_CLOUD_FOLDER = 'point_cloud'
+RGB_FOLDER = "rgb"
+POINT_CLOUD_FOLDER = "point_cloud"
 RADAR_POINT_CLOUD_FOLDER = "radar_point_cloud"
-BOUNDING_BOX_2D_FOLDER = 'bounding_box_2d'
-BOUNDING_BOX_3D_FOLDER = 'bounding_box_3d'
-SEMANTIC_SEGMENTATION_2D_FOLDER = 'semantic_segmentation_2d'
-SEMANTIC_SEGMENTATION_3D_FOLDER = 'semantic_segmentation_3d'
-INSTANCE_SEGMENTATION_2D_FOLDER = 'instance_segmentation_2d'
-INSTANCE_SEGMENTATION_3D_FOLDER = 'instance_segmentation_3d'
-DEPTH_FOLDER = 'depth'
+BOUNDING_BOX_2D_FOLDER = "bounding_box_2d"
+BOUNDING_BOX_3D_FOLDER = "bounding_box_3d"
+SEMANTIC_SEGMENTATION_2D_FOLDER = "semantic_segmentation_2d"
+SEMANTIC_SEGMENTATION_3D_FOLDER = "semantic_segmentation_3d"
+INSTANCE_SEGMENTATION_2D_FOLDER = "instance_segmentation_2d"
+INSTANCE_SEGMENTATION_3D_FOLDER = "instance_segmentation_3d"
+DEPTH_FOLDER = "depth"
 EXTRA_DATA_FOLDER = "extra_data"
 FEATURE_ONTOLOGY_FOLDER = "feature_ontology"
 AGENT_FOLDER = "agent"
 CLASSIFICATION_FOLDER = "classification"
 
 # Scene Directory structure constants
-CALIBRATION_FOLDER = 'calibration'
-ONTOLOGY_FOLDER = 'ontology'
-SCENE_JSON_FILENAME = 'scene.json'
+CALIBRATION_FOLDER = "calibration"
+ONTOLOGY_FOLDER = "ontology"
+SCENE_JSON_FILENAME = "scene.json"
 
 # DGP file naming conventions
 TRI_DGP_SCENE_DATASET_JSON_NAME = "scene_dataset_v{version}.json"
 TRI_DGP_SCENE_JSON_NAME = "scene_{scene_hash}.json"
-ANNOTATION_FILE_NAME = '{image_content_hash}_{annotation_content_hash}.json'
+ANNOTATION_FILE_NAME = "{image_content_hash}_{annotation_content_hash}.json"
 
 # DGP file naming conventions
 TRI_DGP_SCENE_DATASET_JSON_NAME = "scene_dataset_v{version}.json"
 TRI_DGP_AGENT_TRACKS_JSON_NAME = "agent_tracks_{track_hash}.json"
 TRI_DGP_SCENE_JSON_NAME = "scene_{scene_hash}.json"
-ANNOTATION_FILE_NAME = '{image_content_hash}_{annotation_content_hash}.json'
+ANNOTATION_FILE_NAME = "{image_content_hash}_{annotation_content_hash}.json"
 TRI_DGP_AGENTS_JSON_NAME = "agents_{agent_hash}.json"
 TRI_DGP_AGENTS_SLICES_JSON_NAME = "agents_slices_{slice_hash}.json"
 
 # Autolabel constants
-AUTOLABEL_FOLDER = 'autolabels'
-AUTOLABEL_SCENE_JSON_NAME = 'scene.json'
+AUTOLABEL_FOLDER = "autolabels"
+AUTOLABEL_SCENE_JSON_NAME = "scene.json"

dgp/contribs/dgp2wicker/requirements.in

@@ -0,0 +1,3 @@
+-r ../../../requirements.in
+wicker[spark]
+retry

dgp/contribs/dgp2wicker/requirements.txt

@@ -1,2 +1,3 @@
+-r ../../../requirements.txt
 wicker[spark]
 retry

dgp/contribs/dgp2wicker/setup.py

@@ -19,35 +19,34 @@ def run(self):
         develop.run(self)
 
 
-__version__ = importlib.import_module('dgp2wicker').__version__
+__version__ = importlib.import_module("dgp2wicker").__version__
 
-with open('requirements.txt', 'r', encoding='utf-8') as f:
+with open("requirements.txt", "r", encoding="utf-8") as f:
     requirements = f.read().splitlines()
 
-with open('README.md', 'r', encoding='utf-8') as f:
+with open("README.md", "r", encoding="utf-8") as f:
     readme = f.read()
 
-packages = find_packages(exclude=['tests'])
+packages = find_packages(exclude=["tests"])
 setup(
     name="dgp2wicker",
     version=__version__,
     description="Tools to convert TRI's DGP to L5's Wicker format.",
     long_description=readme,
-    long_description_content_type='text/markdown',
+    long_description_content_type="text/markdown",
     author="Chris Ochoa, Kuan Lee",
-    author_email='[email protected], [email protected]',
+    author_email="[email protected], [email protected]",
     url="https://github.com/TRI-ML/dgp/tree/master/dgp/contribs/dgp2wicker",
     packages=packages,
-    entry_points={'console_scripts': [
-        'dgp2wicker=dgp2wicker.cli:cli',
+    entry_points={"console_scripts": [
+        "dgp2wicker=dgp2wicker.cli:cli",
     ]},
     include_package_data=True,
-    setup_requires=['cython==0.29.21', 'grpcio==1.41.0', 'grpcio-tools==1.41.0'],
     install_requires=requirements,
     zip_safe=False,
-    python_requires='>=3.7',
+    python_requires=">=3.8",
     cmdclass={
-        'install': CustomInstallCommand,
-        'develop': CustomDevelopCommand,
-    }
+        "install": CustomInstallCommand,
+        "develop": CustomDevelopCommand,
+    },
 )