Benchmark #16
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # name of the workflow, what it is doing (optional) | |
| name: Benchmark | |
| # events that trigger the workflow (required) | |
| on: | |
| push: | |
| # pushes to the following branches | |
| branches: | |
| - main | |
| pull_request: | |
| # pull request where master is target | |
| branches: | |
| - main | |
| workflow_dispatch: # Add this line to allow manual triggering | |
| jobs: | |
| mot-metrics-benchmark: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] | |
| python-version: ['3.11'] | |
| tracker: ["ocsort", "bytetrack", "botsort", "hybridsort", "deepocsort", "imprassoc", "strongsort"] | |
| timeout-minutes: 50 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: 'pip' | |
| - name: Install requirements | |
| run: | | |
| sudo apt-get install -y jq unzip curl | |
| # run: | | |
| # if [[ "$OSTYPE" == "darwin"* ]]; then | |
| # # macOS | |
| # sed -i '' 's/source="torch_cuda121"/source="torchcpu"/g' pyproject.toml | |
| # elif [[ "$OSTYPE" == "linux-gnu"* ]]; then | |
| # # Linux | |
| # sed -i 's/source="torch_cuda121"/source="torchcpu"/g' pyproject.toml | |
| # fi | |
| # | |
| # python -m pip install --upgrade pip setuptools wheel poetry | |
| # poetry config virtualenvs.create false | |
| # poetry lock --no-update | |
| # poetry install --with yolo | |
| - name: Download eval tools repo | |
| run: | | |
| git clone https://github.com/JonathonLuiten/TrackEval.git tracking/val_utils | |
| # Restore the cached dataset (if available) | |
| - name: Restore MOT17 dataset cache | |
| uses: actions/cache@v3 | |
| with: | |
| # Specify the path where the dataset is stored | |
| path: tracking/val_utils/MOT17.zip | |
| # Create a cache key, you can use a fixed key if the dataset is static | |
| key: mot17-dataset-cache | |
| - name: Download eval data | |
| run: | | |
| wget https://github.com/mikel-brostrom/boxmot/releases/download/v10.0.83/runs.zip -O runs.zip | |
| wget https://github.com/mikel-brostrom/boxmot/releases/download/v10.0.83/MOT17-50.zip -O tracking/val_utils/MOT17.zip | |
| unzip runs.zip -d runs | |
| mkdir -p tracking/val_utils/data/MOT17 | |
| unzip tracking/val_utils/MOT17.zip -d tracking/val_utils/data/MOT17 | |
| # Cache data for future runs (only if the cache was not already restored) | |
| - name: Cache MOT17.zip if not already cached | |
| if: steps.cache-restore.outputs.cache-hit != 'true' # Only run if the cache was not hit | |
| uses: actions/cache@v3 | |
| with: | |
| path: tracking/val_utils/MOT17.zip | |
| key: mot17-dataset-cache | |
| - name: Evaluation and Summarize Results | |
| run: | | |
| if python3 tracking/val.py --imgsz 320 --classes 0 --benchmark MOT17 --yolo-model yolov8x.pt --reid-model osnet_x1_0_dukemtmcreid.pt --tracking-method ${{ matrix.tracker }} --verbose --source ./tracking/val_utils/data/MOT17/train --ci; then | |
| STATUS="✅" | |
| else | |
| STATUS="❌" | |
| fi | |
| if [ -f ${{ matrix.tracker }}_output.json ]; then | |
| HOTA=$(jq -r '.HOTA' ${{ matrix.tracker }}_output.json) | |
| MOTA=$(jq -r '.MOTA' ${{ matrix.tracker }}_output.json) | |
| IDF1=$(jq -r '.IDF1' ${{ matrix.tracker }}_output.json) | |
| else | |
| HOTA="" | |
| MOTA="" | |
| IDF1="" | |
| fi | |
| mkdir results | |
| TRACKER_NAME=$(echo ${{ matrix.tracker }} | awk '{print toupper(substr($0,1,1)) tolower(substr($0,2))}') | |
| echo "$TRACKER_NAME,$STATUS,$HOTA,$MOTA,$IDF1" > results/${{ matrix.tracker }}.txt | |
| - name: Show Results | |
| run: cat results/${{ matrix.tracker }}.txt | |
| - name: Upload Results | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: results | |
| path: results/${{ matrix.tracker }}.txt | |
| combine-results: | |
| runs-on: ubuntu-latest | |
| needs: mot-metrics-benchmark | |
| steps: | |
| - name: Download all results | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: results | |
| path: results | |
| - name: Check downloaded files | |
| run: | | |
| echo "Downloaded files in the results directory:" | |
| ls -la results/ | |
| - name: Combine results | |
| run: | | |
| echo "Format,Status❔,HOTA,MOTA,IDF1" > combined_results.csv | |
| for file in results/*; do | |
| if [ -f "$file" ]; then | |
| cat "$file" >> combined_results.csv # Use cat instead of tail to include all lines | |
| fi | |
| done | |
| # Sort the results by HOTA in descending order | |
| (head -n 1 combined_results.csv && tail -n +2 combined_results.csv | sort -t, -k3 -nr) > sorted_results.csv | |
| # Create a pretty table from the sorted_results.csv file | |
| column -s, -t sorted_results.csv > pretty_results.txt | |
| - name: Show Combined Results | |
| run: cat pretty_results.txt | |
| - name: Upload Combined Results | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: sorted-combined-results | |
| path: pretty_results.txt |