benchmark.yml

# name of the workflow, what it is doing (optional)
name: Benchmark

# events that trigger the workflow (required)
on:
  push:
    # pushes to the following branches
    branches:
      - main
  pull_request:
    # pull request where master is target
    branches:
      - main
  workflow_dispatch:  # Add this line to allow manual triggering


jobs:

  mot-metrics-benchmark:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
        python-version: ['3.11']
        tracker: ["ocsort", "bytetrack", "botsort", "hybridsort", "deepocsort", "imprassoc", "strongsort"]
    timeout-minutes: 50

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'pip'

      - name: Install requirements
        run: |
          sudo apt-get install -y jq curl

          if [[ "$OSTYPE" == "darwin"* ]]; then
            # macOS
            sed -i '' 's/source="torch_cuda121"/source="torchcpu"/g' pyproject.toml
          elif [[ "$OSTYPE" == "linux-gnu"* ]]; then
            # Linux
            sed -i 's/source="torch_cuda121"/source="torchcpu"/g' pyproject.toml
          fi
          
          python -m pip install --upgrade pip setuptools wheel poetry
          poetry config virtualenvs.create false
          poetry lock --no-update
          poetry install --with yolo

      - name: Download eval tools repo
        run: |
          git clone https://github.com/JonathonLuiten/TrackEval.git tracking/val_utils

      # Restore the cached dataset (if available)
      - name: Restore MOT17 dataset cache
        uses: actions/cache@v3
        with:
          # Specify the path where the dataset is stored
          path: tracking/val_utils/MOT17-50.zip
          # Create a cache key, you can use a fixed key if the dataset is static
          key: mot17-50-dataset-cache

      - name: Download eval data
        run: |
          wget https://github.com/mikel-brostrom/boxmot/releases/download/v10.0.83/runs.zip -O runs.zip
          wget https://github.com/mikel-brostrom/boxmot/releases/download/v10.0.83/MOT17-50.zip -O tracking/val_utils/MOT17-50.zip
          unzip runs.zip -d ./
          mkdir -p tracking/val_utils/data
          unzip tracking/val_utils/MOT17-50.zip -d ./tracking/val_utils/data/

      - name: Content
        run: |
          pwd
          cat ./runs/mot/yolov8x_osnet_x1_0_dukemtmcreid_deepocsort/MOT17-02.txt
          ls -l ./runs
          ls -l ./tracking/val_utils/data/
          ls -l ./tracking/val_utils/data/MOT17-50/train

      # Cache data for future runs (only if the cache was not already restored)
      - name: Cache MOT17.zip if not already cached
        if: steps.cache-restore.outputs.cache-hit != 'true'  # Only run if the cache was not hit
        uses: actions/cache@v3
        with:
          path: tracking/val_utils/MOT17-50.zip
          key: mot17-50-dataset-cache

      - name: Evaluation and Summarize Results
        run: |

          if python3 tracking/val.py --imgsz 320 --classes 0 --benchmark MOT17-50 --yolo-model yolov8x.pt --reid-model osnet_x1_0_dukemtmcreid.pt --tracking-method ${{ matrix.tracker }} --verbose --source ./tracking/val_utils/data/MOT17-50/train; then
            STATUS="✅"
          else
            STATUS="❌"
          fi

          if [ -f ${{ matrix.tracker }}_output.json ]; then
            HOTA=$(jq -r '.HOTA' ${{ matrix.tracker }}_output.json)
            MOTA=$(jq -r '.MOTA' ${{ matrix.tracker }}_output.json)
            IDF1=$(jq -r '.IDF1' ${{ matrix.tracker }}_output.json)
          else
            HOTA=""
            MOTA=""
            IDF1=""
          fi

          mkdir results
          TRACKER_NAME=$(echo ${{ matrix.tracker }} | awk '{print toupper(substr($0,1,1)) tolower(substr($0,2))}')
          echo "$TRACKER_NAME,$STATUS,$HOTA,$MOTA,$IDF1" > results/${{ matrix.tracker }}.txt
        
      - name: Show Results
        run: cat results/${{ matrix.tracker }}.txt

      - name: Upload Results
        uses: actions/upload-artifact@v3
        with:
          name: results
          path: results/${{ matrix.tracker }}.txt

  combine-results:
    runs-on: ubuntu-latest
    needs: mot-metrics-benchmark
    steps:
      - uses: actions/checkout@v4

      - name: Download all results
        uses: actions/download-artifact@v3
        with:
          name: results
          path: results

      - name: Check downloaded files
        run: |
          echo "Downloaded files in the results directory:"
          ls -la results/

      - name: Combine results
        run: |
          for file in results/*; do
            if [ -f "$file" ]; then
              cat "$file" >> combined_results.csv  # Use cat instead of tail to include all lines
            fi
          done

          # Sort the results by HOTA in descending order
          (head -n 1 combined_results.csv && tail -n +2 combined_results.csv | sort -t, -k3 -nr) > sorted_results.csv

          # Create a pretty table from the sorted_results.csv file
          column -s, -t sorted_results.csv > pretty_results.txt

      - name: Show Combined Results
        run: cat pretty_results.txt

      - name: Set up Git
        run: |
          git config --local user.email "yolov5.deepsort.pytorch@gmail.com"
          git config --local user.name "mikel-brostrom"

      - name: Update README with tracker results
        run: |
          # Paths
        RESULTS_FILE="pretty_results.txt"
        README_FILE="README.md"

        # Backup original README in case needed
        cp "$README_FILE" "${README_FILE}.bak"

        # Create the markdown table header
        new_table="| Tracker | Status  | HOTA↑ | MOTA↑ | IDF1↑ |\n"
        new_table+="| ------- | ------- | ----- | ----- | ----- |\n"

        # Append the contents of pretty_results.txt to the table
        while read -r line; do
            new_table+="| $(echo "$line" | awk '{print $1}') | $(echo "$line" | awk '{print $2}') | $(echo "$line" | awk '{print $3}') | $(echo "$line" | awk '{print $4}') | $(echo "$line" | awk '{print $5}') |\n"
        done < "$RESULTS_FILE"

        # Define unique markers to locate the table
        start_marker="<!-- START TRACKER TABLE -->"
        end_marker="<!-- END TRACKER TABLE -->"

        # Use awk to replace lines between markers, preserving non-table content
        awk -v start_marker="$start_marker" -v end_marker="$end_marker" -v new_table="$new_table" '
            BEGIN { in_table=0 }
            {
                if ($0 == start_marker) {
                    print $0      # Print the start marker
                    print new_table  # Insert the new table
                    in_table=1    # Start skipping lines in the old table
                    next
                }
                else if ($0 == end_marker) {
                    in_table=0    # End skipping when reaching end_marker
                    print $0      # Print end marker
                    next
                }
                if (!in_table) {
                    print $0  # Continue printing lines outside the table
                }
            }
        ' "$README_FILE" > temp_readme.md

        # Replace original README with updated version
        mv temp_readme.md "$README_FILE"

      # Check for changes
      - name: Check for changes
        id: check_changes
        run: |
          if git diff --quiet; then
            echo "No changes to commit."
            echo "::set-output name=changed::false"
          else
            echo "::set-output name=changed::true"
          fi

      # Commit and push changes if there are any
      - name: Commit changes
        if: steps.check_changes.outputs.changed == 'true'
        run: |
          BRANCH_NAME="update-tracker-results-$(date +%Y%m%d%H%M%S)"
          git checkout -b "$BRANCH_NAME"
          git add README.md
          git commit -m "Overwrote tracker results in README.md"
          git push origin "$BRANCH_NAME"

      # Create a pull request (PR)
      - name: Create Pull Request
        if: steps.check_changes.outputs.changed == 'true'
        uses: peter-evans/create-pull-request@v5
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          branch: ${{ github.ref }}
          commit-message: Overwrote tracker results in README.md
          title: Overwrite tracker results in README.md
          body: "This PR updates the tracker results table in the README.md."