test: rerun only failed test on action

.github/scripts/run-maestro.sh

@@ -0,0 +1,199 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+PLATFORM="${1:-${PLATFORM:-android}}"
+SHARD="${2:-${SHARD:-default}}"
+FLOWS_DIR=".maestro/tests"
+MAIN_REPORT="maestro-report.xml"
+MAX_RERUN_ROUNDS="${MAX_RERUN_ROUNDS:-2}"
+RERUN_REPORT_PREFIX="maestro-rerun"
+export MAESTRO_DRIVER_STARTUP_TIMEOUT="${MAESTRO_DRIVER_STARTUP_TIMEOUT:-120000}"
+
+if ! command -v maestro >/dev/null 2>&1; then
+  echo "ERROR: maestro not found in PATH"
+  exit 2
+fi
+
+if [ "$PLATFORM" = "android" ]; then
+  if ! command -v adb >/dev/null 2>&1; then
+    echo "ERROR: adb not found"
+    exit 2
+  fi
+else
+  if ! command -v xcrun >/dev/null 2>&1; then
+    echo "ERROR: xcrun not found"
+    exit 2
+  fi
+fi
+
+MAPFILE="$(mktemp)"
+trap 'rm -f "$MAPFILE"' EXIT
+
+while IFS= read -r -d '' file; do
+  if grep -qE "^[[:space:]]*-[[:space:]]*['\"]?test-${SHARD}['\"]?([[:space:]]*$|[[:space:]]*,|[[:space:]]*\\])" "$file"; then
+    raw_name="$(grep -m1 -E '^[[:space:]]*name:' "$file" || true)"
+    if [ -n "$raw_name" ]; then
+      name_val="$(echo "$raw_name" | sed -E 's/^[[:space:]]*name:[[:space:]]*//; s/^["'\'']//; s/["'\'']$//; s/[[:space:]]*$//')"
+      name_val="$(echo "$name_val" | sed -E 's/^[[:space:]]+//; s/[[:space:]]+$//')"
+      if [ -n "$name_val" ]; then
+        printf '%s\t%s\n' "$name_val" "$file" >> "$MAPFILE"
+      fi
+    fi
+  fi
+done < <(find "$FLOWS_DIR" -type f \( -iname '*.yml' -o -iname '*.yaml' \) -print0)
+
+if [ ! -s "$MAPFILE" ]; then
+  echo "No flows for test-${SHARD}"
+  exit 1
+fi
+
+echo "Mapped flows for tag test-${SHARD}:"
+awk -F'\t' '{ printf "  %s -> %s\n", $1, $2 }' "$MAPFILE"
+
+FLOW_FILES=()
+declare -A SEEN
+while IFS=$'\t' read -r name path; do
+  if [ -z "${SEEN[$path]:-}" ]; then
+    FLOW_FILES+=("$path")
+    SEEN[$path]=1
+  fi
+done < "$MAPFILE"
+
+echo "Main run will execute:"
+printf '  %s\n' "${FLOW_FILES[@]}"
+
+if [ "$PLATFORM" = "android" ]; then
+  adb shell settings put system show_touches 1 || true
+  adb install -r "app-experimental-release.apk" || true
+  adb shell monkey -p "chat.rocket.reactnative" -c android.intent.category.LAUNCHER 1 || true
+  sleep 6
+  adb shell am force-stop "chat.rocket.reactnative" || true
+
+  maestro test "${FLOW_FILES[@]}" \
+    --exclude-tags=util \
+    --include-tags="test-${SHARD}" \
+    --format junit \
+    --output "$MAIN_REPORT" || true
+
+else
+  maestro test "${FLOW_FILES[@]}" \
+    --exclude-tags=util \
+    --include-tags="test-${SHARD}" \
+    --exclude-tags=android-only \
+    --format junit \
+    --output "$MAIN_REPORT" || true
+fi
+
+if [ ! -f "$MAIN_REPORT" ]; then
+  echo "Main report not found"
+  exit 1
+fi
+
+FAILED_NAMES="$(python3 - <<PY
+import sys,xml.etree.ElementTree as ET
+try:
+  tree = ET.parse("$MAIN_REPORT")
+except:
+  sys.exit(0)
+root = tree.getroot()
+failed=[]
+for tc in root.findall(".//testcase"):
+  if tc.find("failure") is not None or tc.find("error") is not None:
+    if tc.get("name"):
+      failed.append(tc.get("name").strip())
+for n in sorted(set(failed)):
+  print(n)
+PY
+)"
+
+if [ -z "$FAILED_NAMES" ]; then
+  echo "All tests passed."
+  exit 0
+fi
+
+IFS=$'\n' read -rd '' -a FAILED_ARRAY <<<"$FAILED_NAMES" || true
+
+CANDIDATE_FILES=()
+SEEN2=""
+for NAME in "${FAILED_ARRAY[@]}"; do
+  FILE="$(awk -F'\t' -v n="$NAME" '$1==n {print $2; exit}' "$MAPFILE" || true)"
+  if [ -n "$FILE" ] && ! printf '%s\n' "$SEEN2" | grep -Fq "$FILE"; then
+    CANDIDATE_FILES+=("$FILE")
+    SEEN2="${SEEN2}"$'\n'"${FILE}"
+  fi
+done
+
+if [ ${#CANDIDATE_FILES[@]} -eq 0 ]; then
+  echo "No flow files to retry"
+  exit 1
+fi
+
+CURRENT_FAILS=("${CANDIDATE_FILES[@]}")
+ROUND=1
+
+while [ ${#CURRENT_FAILS[@]} -gt 0 ] && [ "$ROUND" -le "$MAX_RERUN_ROUNDS" ]; do
+  echo "=== RERUN ROUND $ROUND (${#CURRENT_FAILS[@]} flows) ==="
+
+  RPT="${RERUN_REPORT_PREFIX}-round-${ROUND}.xml"
+
+  if [ "$PLATFORM" = "android" ]; then
+    maestro test "${CURRENT_FAILS[@]}" \
+      --exclude-tags=util \
+      --include-tags="test-${SHARD}" \
+      --format junit \
+      --output "$RPT" || true
+  else
+    maestro test "${CURRENT_FAILS[@]}" \
+      --exclude-tags=util \
+      --include-tags="test-${SHARD}" \
+      --exclude-tags=android-only \
+      --format junit \
+      --output "$RPT" || true
+  fi
+
+  if [ ! -f "$RPT" ]; then
+    echo "Rerun report missing"
+    break
+  fi
+
+  NEXT_FAILED="$(python3 - <<PY
+import sys,xml.etree.ElementTree as ET
+try:
+  tree = ET.parse("$RPT")
+except:
+  sys.exit(0)
+root = tree.getroot()
+failed=[]
+for tc in root.findall(".//testcase"):
+  if tc.find("failure") is not None or tc.find("error") is not None:
+    if tc.get("name"):
+      failed.append(tc.get("name").strip())
+for n in sorted(set(failed)):
+  print(n)
+PY
+)"
+
+  if [ -z "$NEXT_FAILED" ]; then
+    echo "All retried flows passed in this round."
+    exit 0
+  fi
+
+  IFS=$'\n' read -rd '' -a NEXT_FAILED_ARRAY <<<"$NEXT_FAILED" || true
+
+  NEXT_FILES=()
+  SEEN3=""
+  for NAME in "${NEXT_FAILED_ARRAY[@]}"; do
+    FILE="$(awk -F'\t' -v n="$NAME" '$1==n {print $2; exit}' "$MAPFILE" || true)"
+    if [ -n "$FILE" ] && ! printf '%s\n' "$SEEN3" | grep -Fq "$FILE"; then
+      NEXT_FILES+=("$FILE")
+      SEEN3="${SEEN3}"$'\n'"${FILE}"
+    fi
+  done
+
+  CURRENT_FAILS=("${NEXT_FILES[@]}")
+  ROUND=$((ROUND+1))
+done
+
+echo "Retry strategy finished with remaining failures:"
+printf '%s\n' "${CURRENT_FAILS[@]}"
+exit 1

.github/workflows/maestro-android.yml

@@ -43,63 +43,12 @@ jobs:
           sudo udevadm control --reload-rules
           sudo udevadm trigger --name-match=kvm
 
-      - name: Create Maestro script
-        run: |
-          cat << 'EOF' > run-maestro.sh
-          #!/bin/bash
-          SHARD=${{ inputs.shard }}
-          echo "Running shard: $SHARD"
-
-          adb shell settings put system show_touches 1
-          adb install app-experimental-release.apk
-          adb shell monkey -p chat.rocket.reactnative -c android.intent.category.LAUNCHER 1
-          sleep 10
-          adb shell am force-stop chat.rocket.reactnative
-          export MAESTRO_DRIVER_STARTUP_TIMEOUT=120000
-
-          MAX_RETRIES=3
-          ATTEMPT=1
-          FINAL_EXIT_CODE=1
-
-          while [ $ATTEMPT -le $MAX_RETRIES ]; do
-            echo "Attempt $ATTEMPT of $MAX_RETRIES"
-
-            echo "Starting screen recording..."
-            adb shell screenrecord /sdcard/test_run.mp4 > /dev/null 2>&1 &
-            RECORD_PID=$!
-
-            maestro test .maestro --exclude-tags=util --include-tags=test-$SHARD --format junit --output maestro-report.xml
-            TEST_EXIT_CODE=$?
-
-            echo "Stopping screen recording..."
-            kill -INT $RECORD_PID || true
-            sleep 2
-
-            echo "Pulling video from device..."
-            adb pull /sdcard/test_run.mp4 test_run_${SHARD}_attempt_${ATTEMPT}.mp4 || true
-            adb shell rm /sdcard/test_run.mp4 || true
-
-            if [ $TEST_EXIT_CODE -eq 0 ]; then
-              echo "Maestro passed on attempt $ATTEMPT"
-              FINAL_EXIT_CODE=0
-              break
-            else
-              echo "Maestro failed on attempt $ATTEMPT"
-            fi
-
-            ATTEMPT=$((ATTEMPT+1))
-          done
-
-          exit $FINAL_EXIT_CODE
-          EOF
-
-          chmod +x run-maestro.sh
-        env:
-          SHARD: ${{ inputs.shard }}
+      - name: Make Maestro script executable
+        run: chmod +x .github/scripts/run-maestro.sh
 
       - name: Start Android Emulator and Run Maestro Tests
         uses: reactivecircus/android-emulator-runner@v2
-        timeout-minutes: 60
+        timeout-minutes: 120
         with:
           api-level: 34
           disk-size: 4096M
@@ -111,20 +60,12 @@ jobs:
           force-avd-creation: false
           emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
           disable-animations: true
-          script: ./run-maestro.sh
+          script: ./.github/scripts/run-maestro.sh android ${{ inputs.shard }}
 
-      - name: Upload Test Report
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: Android Test Report - Shard ${{ inputs.shard }}
-          path: maestro-report.xml
-          retention-days: 7
-
-      - name: Upload Screen Recording
+      - name: Android Maestro Logs
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: maestro-video-${{ inputs.shard }}
-          path: test_run_${{ inputs.shard }}_attempt_*.mp4
+          name: Android Maestro Logs - Shard ${{ inputs.shard }}
+          path: ~/.maestro/tests/**/*.png
           retention-days: 7

.github/workflows/maestro-ios.yml

@@ -84,40 +84,21 @@ jobs:
           echo "UDID=$UDID"
           echo "UDID=$UDID" >> $GITHUB_ENV
 
+      - name: Make Maestro script executable
+        run: chmod +x .github/scripts/run-maestro.sh
+
       - name: Install App
         run: |
           xcrun simctl install $UDID "ios-simulator-app"
 
       - name: Run Tests
-        id: maestro-tests
-        uses: nick-fields/retry@v3
-        with:
-          max_attempts: 3
-          timeout_minutes: 60
-          command: |
-            SHARD=${{ inputs.shard }}
-            echo "Running shard: $SHARD"
-            export MAESTRO_DRIVER_STARTUP_TIMEOUT=120000
-
-            maestro test .maestro \
-              --exclude-tags=util \
-              --include-tags=test-$SHARD \
-              --exclude-tags=android-only \
-              --format junit \
-              --output maestro-report.xml
-
-      - name: Maestro Test Report
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: Test Report - Shard ${{ inputs.shard }}
-          path: maestro-report.xml
-          retention-days: 28
+        timeout-minutes: 120
+        run: ./.github/scripts/run-maestro.sh ios ${{ inputs.shard }}
 
       - name: iOS Maestro Logs
         if: always()
         uses: actions/upload-artifact@v4
         with:
           name: iOS Maestro Logs - Shard ${{ inputs.shard }}
-          path: ~/.maestro/tests/
-          retention-days: 28
+          path: ~/.maestro/tests/**/*.png
+          retention-days: 7