Benchmark #31
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # name of the workflow, what it is doing (optional) | |
| name: Benchmark | |
| # events that trigger the workflow (required) | |
| on: | |
| push: | |
| # pushes to the following branches | |
| branches: | |
| - main | |
| pull_request: | |
| # pull request where master is target | |
| branches: | |
| - main | |
| workflow_dispatch: # Add this line to allow manual triggering | |
| jobs: | |
| mot-metrics-benchmark: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] | |
| python-version: ['3.11'] | |
| tracker: ["ocsort", "bytetrack", "botsort", "hybridsort", "deepocsort", "imprassoc", "strongsort"] | |
| timeout-minutes: 50 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: 'pip' | |
| - name: Install requirements | |
| run: | | |
| sudo apt-get install -y jq curl | |
| if [[ "$OSTYPE" == "darwin"* ]]; then | |
| # macOS | |
| sed -i '' 's/source="torch_cuda121"/source="torchcpu"/g' pyproject.toml | |
| elif [[ "$OSTYPE" == "linux-gnu"* ]]; then | |
| # Linux | |
| sed -i 's/source="torch_cuda121"/source="torchcpu"/g' pyproject.toml | |
| fi | |
| python -m pip install --upgrade pip setuptools wheel poetry | |
| poetry config virtualenvs.create false | |
| poetry lock --no-update | |
| poetry install --with yolo | |
| - name: Download eval tools repo | |
| run: | | |
| git clone https://github.com/JonathonLuiten/TrackEval.git tracking/val_utils | |
| # Restore the cached dataset (if available) | |
| - name: Restore MOT17 dataset cache | |
| uses: actions/cache@v3 | |
| with: | |
| # Specify the path where the dataset is stored | |
| path: tracking/val_utils/MOT17-50.zip | |
| # Create a cache key, you can use a fixed key if the dataset is static | |
| key: mot17-50-dataset-cache | |
| - name: Download eval data | |
| run: | | |
| wget https://github.com/mikel-brostrom/boxmot/releases/download/v10.0.83/runs.zip -O runs.zip | |
| wget https://github.com/mikel-brostrom/boxmot/releases/download/v10.0.83/MOT17-50.zip -O tracking/val_utils/MOT17-50.zip | |
| unzip runs.zip -d ./ | |
| mkdir -p tracking/val_utils/data | |
| unzip tracking/val_utils/MOT17-50.zip -d ./tracking/val_utils/data/ | |
| - name: Content | |
| run: | | |
| pwd | |
| cat ./runs/mot/yolov8x_osnet_x1_0_dukemtmcreid_deepocsort/MOT17-02.txt | |
| ls -l ./runs | |
| ls -l ./tracking/val_utils/data/ | |
| ls -l ./tracking/val_utils/data/MOT17-50/train | |
| # Cache data for future runs (only if the cache was not already restored) | |
| - name: Cache MOT17.zip if not already cached | |
| if: steps.cache-restore.outputs.cache-hit != 'true' # Only run if the cache was not hit | |
| uses: actions/cache@v3 | |
| with: | |
| path: tracking/val_utils/MOT17-50.zip | |
| key: mot17-50-dataset-cache | |
| - name: Evaluation and Summarize Results | |
| run: | | |
| if python3 tracking/val.py --imgsz 320 --classes 0 --benchmark MOT17-50 --yolo-model yolov8x.pt --reid-model osnet_x1_0_dukemtmcreid.pt --tracking-method ${{ matrix.tracker }} --verbose --source ./tracking/val_utils/data/MOT17-50/train; then | |
| STATUS="✅" | |
| else | |
| STATUS="❌" | |
| fi | |
| if [ -f ${{ matrix.tracker }}_output.json ]; then | |
| HOTA=$(jq -r '.HOTA' ${{ matrix.tracker }}_output.json) | |
| MOTA=$(jq -r '.MOTA' ${{ matrix.tracker }}_output.json) | |
| IDF1=$(jq -r '.IDF1' ${{ matrix.tracker }}_output.json) | |
| else | |
| HOTA="" | |
| MOTA="" | |
| IDF1="" | |
| fi | |
| mkdir results | |
| TRACKER_NAME=$(echo ${{ matrix.tracker }} | awk '{print toupper(substr($0,1,1)) tolower(substr($0,2))}') | |
| echo "$TRACKER_NAME,$STATUS,$HOTA,$MOTA,$IDF1" > results/${{ matrix.tracker }}.txt | |
| - name: Show Results | |
| run: cat results/${{ matrix.tracker }}.txt | |
| - name: Upload Results | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: results | |
| path: results/${{ matrix.tracker }}.txt | |
| combine-results: | |
| runs-on: ubuntu-latest | |
| needs: mot-metrics-benchmark | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download all results | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: results | |
| path: results | |
| - name: Check downloaded files | |
| run: | | |
| echo "Downloaded files in the results directory:" | |
| ls -la results/ | |
| - name: Combine results | |
| run: | | |
| echo "Format,Status❔,HOTA,MOTA,IDF1" > combined_results.csv | |
| for file in results/*; do | |
| if [ -f "$file" ]; then | |
| cat "$file" >> combined_results.csv # Use cat instead of tail to include all lines | |
| fi | |
| done | |
| # Sort the results by HOTA in descending order | |
| (head -n 1 combined_results.csv && tail -n +2 combined_results.csv | sort -t, -k3 -nr) > sorted_results.csv | |
| # Create a pretty table from the sorted_results.csv file | |
| column -s, -t sorted_results.csv > pretty_results.txt | |
| - name: Show Combined Results | |
| run: cat pretty_results.txt | |
| - name: Set up Git | |
| run: | | |
| git config --local user.email "[email protected]" | |
| git config --local user.name "mikel-brostrom" | |
| - name: Update README with tracker results | |
| run: | | |
| # Paths | |
| RESULTS_FILE="pretty_results.txt" | |
| README_FILE="README.md" | |
| # Backup original README in case needed | |
| cp "$README_FILE" "${README_FILE}.bak" | |
| # Extract the new table from the pretty_results.txt file | |
| new_table="| Tracker | HOTA↑ | MOTA↑ | IDF1↑ |\n" | |
| new_table+="| -------- | ----- | ----- | ----- |\n" | |
| while read -r line; do | |
| # Match lines with tracker data and handle missing values | |
| if [[ $line =~ ^(Strongsort|Ocsort|Deepocsort|Bytetrack|Botsort|Imprassoc|Hybridsort)[[:space:]]+([[:graph:]]*)[[:space:]]+([[:graph:]]*)[[:space:]]+([[:graph:]]*) ]]; then | |
| tracker=${BASH_REMATCH[1]} | |
| hota=${BASH_REMATCH[2]:-N/A} # Default to N/A if missing | |
| mota=${BASH_REMATCH[3]:-N/A} # Default to N/A if missing | |
| idf1=${BASH_REMATCH[4]:-N/A} # Default to N/A if missing | |
| if [[ "$tracker" == "Imprassoc" ]]; then | |
| new_table+="| [$tracker](https://ieeexplore.ieee.org/document/10223159) | $hota | $mota | $idf1 |\n" | |
| else | |
| new_table+="| [$tracker](https://arxiv.org/pdf/$tracker.pdf) | $hota | $mota | $idf1 |\n" | |
| fi | |
| fi | |
| done < "$RESULTS_FILE" | |
| # Add any additional rows (like image placeholders) if needed | |
| new_table+="| <img width=200/> | <img width=100/> | <img width=100/> | <img width=100/> |\n" | |
| # Define unique markers to locate the table | |
| start_marker="<!-- START TRACKER TABLE -->" | |
| end_marker="<!-- END TRACKER TABLE -->" | |
| # Use awk to replace lines between markers, preserving non-table content | |
| awk -v start_marker="$start_marker" -v end_marker="$end_marker" -v new_table="$new_table" ' | |
| BEGIN { in_table=0 } | |
| { | |
| if ($0 == start_marker) { | |
| print $0 # Print the start marker | |
| print new_table # Insert the new table | |
| in_table=1 # Start skipping lines in the old table | |
| next | |
| } | |
| else if ($0 == end_marker) { | |
| in_table=0 # End skipping when reaching end_marker | |
| print $0 # Print end marker | |
| next | |
| } | |
| if (!in_table) { | |
| print $0 # Continue printing lines outside the table | |
| } | |
| } | |
| ' "$README_FILE" > temp_readme.md | |
| # Replace original README with updated version | |
| mv temp_readme.md "$README_FILE" | |
| # Check for changes | |
| - name: Check for changes | |
| id: check_changes | |
| run: | | |
| if git diff --quiet; then | |
| echo "No changes to commit." | |
| echo "::set-output name=changed::false" | |
| else | |
| echo "::set-output name=changed::true" | |
| fi | |
| # Commit and push changes if there are any | |
| - name: Commit changes | |
| if: steps.check_changes.outputs.changed == 'true' | |
| run: | | |
| BRANCH_NAME="update-tracker-results-$(date +%Y%m%d%H%M%S)" | |
| git checkout -b "$BRANCH_NAME" | |
| git add README.md | |
| git commit -m "Overwrote tracker results in README.md" | |
| git push origin "$BRANCH_NAME" | |
| # Create a pull request (PR) | |
| - name: Create Pull Request | |
| if: steps.check_changes.outputs.changed == 'true' | |
| uses: peter-evans/create-pull-request@v5 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| branch: ${{ github.ref }} | |
| commit-message: Overwrote tracker results in README.md | |
| title: Overwrite tracker results in README.md | |
| body: "This PR updates the tracker results table in the README.md." |