benchmark.yml

# name of the workflow, what it is doing (optional)
name: Benchmark

# events that trigger the workflow (required)
on:
  push:
    # pushes to the following branches
    branches:
      - main
  pull_request:
    # pull request where master is target
    branches:
      - main
  workflow_dispatch:  # Add this line to allow manual triggering

permissions:
  contents: write        # Allows reading and writing repository contents
  pull-requests: write

jobs:
  mot-metrics-benchmark:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
        python-version: ['3.11']
        tracker: ["ocsort", "bytetrack", "botsort", "hybridsort", "deepocsort", "imprassoc", "strongsort"]
    timeout-minutes: 50

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'pip'

      - name: Install requirements
        run: |
          sudo apt-get install -y jq curl

          if [[ "$OSTYPE" == "darwin"* ]]; then
            # macOS
            sed -i '' 's/source="torch_cuda121"/source="torchcpu"/g' pyproject.toml
          elif [[ "$OSTYPE" == "linux-gnu"* ]]; then
            # Linux
            sed -i 's/source="torch_cuda121"/source="torchcpu"/g' pyproject.toml
          fi
          
          python -m pip install --upgrade pip setuptools wheel poetry
          poetry config virtualenvs.create false
          poetry lock
          poetry install --with yolo

      - name: Download eval tools repo
        run: |
          git clone https://github.com/JonathonLuiten/TrackEval.git tracking/val_utils

      # Restore the cached dataset (if available)
      - name: Restore MOT17 dataset cache
        uses: actions/cache@v4
        with:
          # Specify the path where the dataset is stored
          path: tracking/val_utils/MOT17-50.zip
          # Create a cache key, you can use a fixed key if the dataset is static
          key: mot17-50-dataset-cache-v3

      - name: Download eval data
        run: |
          wget https://github.com/mikel-brostrom/boxmot/releases/download/v10.0.83/runs.zip -O runs.zip
          wget https://github.com/mikel-brostrom/boxmot/releases/download/v10.0.83/MOT17-50.zip -O tracking/val_utils/MOT17-50.zip
          unzip runs.zip -d ./
          mkdir -p tracking/val_utils/data
          unzip tracking/val_utils/MOT17-50.zip -d ./tracking/val_utils/data/

      # Cache data for future runs (only if the cache was not already restored)
      - name: Cache MOT17.zip if not already cached
        if: steps.cache-restore.outputs.cache-hit != 'true'  # Only run if the cache was not hit
        uses: actions/cache@v4
        with:
          path: tracking/val_utils/MOT17-50.zip
          key: mot17-50-dataset-cache-v3

      - name: Evaluation and Summarize Results
        run: |

          pwd
          ls tracking/val_utils
          ls tracking/val_utils/data

          if python3 tracking/val.py --imgsz 320 --classes 0 --yolo-model yolov8x.pt --reid-model osnet_x1_0_dukemtmcreid.pt --tracking-method ${{ matrix.tracker }} --verbose --source ./tracking/val_utils/data/MOT17-50/train; then
            STATUS="✅"
          else
            STATUS="❌"
          fi

          if [ -f ${{ matrix.tracker }}_output.json ]; then
            HOTA=$(jq -r '.HOTA' ${{ matrix.tracker }}_output.json)
            MOTA=$(jq -r '.MOTA' ${{ matrix.tracker }}_output.json)
            IDF1=$(jq -r '.IDF1' ${{ matrix.tracker }}_output.json)
          else
            HOTA=""
            MOTA=""
            IDF1=""
          fi

          mkdir results
          TRACKER_NAME=$(echo "${{ matrix.tracker }}" | awk '{print tolower($0)}')
          echo "$TRACKER_NAME,$STATUS,$HOTA,$MOTA,$IDF1" > results/${{ matrix.tracker }}.txt
        
      - name: Show Results
        run: cat results/${{ matrix.tracker }}.txt

      - name: Upload Results
        uses: actions/upload-artifact@v4
        with:
          name: results-${{ github.run_id }}-${{ matrix.tracker }}
          path: results/${{ matrix.tracker }}.txt

  combine-results:
    runs-on: ubuntu-latest
    needs: mot-metrics-benchmark
    steps:
      - uses: actions/checkout@v4

      - name: Download all results artifacts
        uses: actions/download-artifact@v4
        with:
          path: results  # Specify the path where you want to store the artifacts
          run-id: ${{ github.run_id }}

      - name: Check downloaded files
        run: |
          echo "Downloaded files in the results directory:"
          ls -la results/*/

      - name: Combine results
        run: |
          touch combined_results.csv  # Ensure the file exists
          for file in results/*/*; do
            if [ -f "$file" ]; then
              cat "$file" >> combined_results.csv  # Use cat to include all lines
            fi
          done

          # Sort the results by the third column (HOTA) in descending order
          sort -t, -k3 -nr combined_results.csv > sorted_results.csv

          # Create a pretty table from the sorted_results.csv file
          column -s, -t sorted_results.csv > pretty_results.txt

      - name: Show Combined Results
        run: cat pretty_results.txt

      - name: Set up Git
        run: |
          git config --local user.email "yolov5.deepsort.pytorch@gmail.com"
          git config --local user.name "mikel-brostrom"

      - name: Update README with tracker results
        run: |
          # Paths
          RESULTS_FILE="pretty_results.txt"
          README_FILE="README.md"

          # Backup original README in case needed
          cp "$README_FILE" "${README_FILE}.bak"

          # Define paper names and their corresponding URLs
          declare -A paper_links
          paper_links["deepocsort"]="https://arxiv.org/abs/2302.11813"
          paper_links["bytetrack"]="https://arxiv.org/abs/2110.06864"
          paper_links["ocsort"]="https://arxiv.org/abs/2203.14360"
          paper_links["strongsort"]="https://arxiv.org/abs/2202.13514"
          paper_links["botsort"]="https://arxiv.org/abs/2206.14651"
          paper_links["hybridsort"]="https://arxiv.org/abs/2308.00783"
          paper_links["imprassoc"]="https://openaccess.thecvf.com/content/CVPR2023W/E2EAD/papers/Stadler_An_Improved_Association_Pipeline_for_Multi-Person_Tracking_CVPRW_2023_paper.pdf"


          # Create the markdown table header
          new_table="| Tracker | Status  | HOTA↑ | MOTA↑ | IDF1↑ |\n"
          new_table+="| :-----: | :-----: | :---: | :---: | :---: |\n"

          # Append the contents of pretty_results.txt to the table
          while read -r line; do
              tracker=$(echo "$line" | awk '{print $1}')
              status=$(echo "$line" | awk '{print $2}')
              hota=$(echo "$line" | awk '{print $3}')
              mota=$(echo "$line" | awk '{print $4}')
              idf1=$(echo "$line" | awk '{print $5}')
              
              # Retrieve the paper URL for the tracker
              paper_url=${paper_links[$tracker]}
              
              # Create a markdown hyperlink for the tracker name
              tracker_link="[$tracker]($paper_url)"
              
              new_table+="| $tracker_link | $status | $hota | $mota | $idf1 |\n"
          done < "$RESULTS_FILE"

          # Define unique markers to locate the table
          start_marker="<!-- START TRACKER TABLE -->"
          end_marker="<!-- END TRACKER TABLE -->"

          # Use awk to replace lines between markers, preserving non-table content
          awk -v start_marker="$start_marker" -v end_marker="$end_marker" -v new_table="$new_table" '
              $0 == start_marker { print $0; print new_table; in_table=1; next }
              $0 == end_marker { in_table=0; print $0; next }
              !in_table
          ' "$README_FILE" > temp_readme.md

          # Replace original README with updated version
          mv temp_readme.md "$README_FILE"

      # Check for changes
      - name: Check for changes in README.md
        id: check_changes
        run: |
          if git diff --quiet README.md; then
            echo "No changes to commit."
            echo "changed=false" >> $GITHUB_ENV  # Writing to the environment file
          else
            echo "README changed!"
            echo "changed=true" >> $GITHUB_ENV  # Writing to the environment file
            BRANCH_NAME="update-tracker-results-$(date +%Y%m%d%H%M%S)"
            echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV  # Store the branch name in the GitHub environment
          fi

      # Create a pull request (PR)
      - name: Create Pull Request
        if: env.changed == 'true'
        uses: peter-evans/create-pull-request@v7
        with:
          add-paths: README.md
          token: ${{ secrets.GITHUB_TOKEN }}
          branch: ${{ env.BRANCH_NAME }}  # Use the created branch
          base: master  # Base branch for the pull request, replace with your actual base branch (e.g., main or develop)
          commit-message: Overwrite tracker results in README.md
          title: Overwrite tracker results in README.md
          body: "This PR updates the tracker results table in the README.md."