Patch for <macOS 15 build systems (#283)

* Prevent units tests from succeeding on error

* Use compiler flags for mltensor sampling

* Add flag for mltensor utils

* Update platform versions for development and pre-release tests workflows

* Differentiate artifact name in unit-tests.yml workflow

* Stop unit tests early if run condition is false

* Fix macos 14 runner ios version

* Use success error code for expected test skipping

* Lower priority of early stopping task, cleanup

* Formatting

* Fix tests, attempt to lower early stopping task priority further

* Fix inverted action run condition logic

* Use detached lower priority for early stopping to resolve priority inversion

* Fix tests

* Set test priority for early stopping, fix correctness test on macos 14

* Upgrade unit test task priority

* Specify device for older iOS simulators

* Fix workflow for ios-device

* Disable ealy stopping test on watchos

* Set xcode version on CI

* Make sure test simulator is available on runner

Author: ZachNagengast

.github/workflows/development-tests.yml

@@ -15,7 +15,8 @@ jobs:
     name: "Build and Test"
     uses: ./.github/workflows/unit-tests.yml
     with:
-      ios-version: "18.1"
+      ios-version: "18.2"
+      ios-device: "iPhone 16"
       macos-runner: "macos-15"
 
   check-approvals:
@@ -42,7 +43,20 @@ jobs:
     name: "Pre-merge Tests"
     needs: [check-approvals]
     if: needs.check-approvals.outputs.reviews == 'APPROVED' || github.event_name == 'workflow_dispatch'
+    strategy:
+      matrix:
+        include:
+          - os: macos-13-xlarge
+            ios-version: "17.2"
+            ios-device: "iPhone 14"
+            xcode-version: "15.2"
+          - os: macos-14
+            ios-version: "17.2"
+            ios-device: "iPhone 15"
+            xcode-version: "15.2"
     uses: ./.github/workflows/unit-tests.yml
     with:
-      ios-version: "16.1"
-      macos-runner: "macos-13-xlarge"
+      macos-runner: ${{ matrix.os }}
+      ios-version: ${{ matrix.ios-version }}
+      ios-device: ${{ matrix.ios-device }}
+      xcode-version: ${{ matrix.xcode-version }}

.github/workflows/pre-release-tests.yml

@@ -12,10 +12,20 @@ jobs:
       matrix:
         include:
           - os: macos-13-xlarge
-            ios-version: "16.1" # Oldest available version
+            ios-version: "17.2" # TODO: Download older simulators for macOS 13
+            ios-device: "iPhone 14"
+            xcode-version: "15.2"
+          - os: macos-14
+            ios-version: "17.2"
+            ios-device: "iPhone 15"
+            xcode-version: "15.2"
           - os: macos-15
-            ios-version: "18.1" # Latest available version
+            ios-version: "18.2" # Latest available version
+            ios-device: "iPhone 16"
+            xcode-version: "latest-stable"
     uses: ./.github/workflows/unit-tests.yml
     with:
-      ios-version: ${{ matrix.ios-version }}
       macos-runner: ${{ matrix.os }}
+      ios-version: ${{ matrix.ios-version }}
+      ios-device: ${{ matrix.ios-device }}
+      xcode-version: ${{ matrix.xcode-version }}

.github/workflows/unit-tests.yml

@@ -6,9 +6,15 @@ on:
       ios-version:
         required: true
         type: string
+      ios-device:
+        required: true
+        type: string
       macos-runner:
         required: true
         type: string
+      xcode-version:
+        required: false
+        type: string
 
 jobs:
   unit-tests:
@@ -27,7 +33,7 @@ jobs:
               name: "iOS",
               condition: true,
               clean-destination: "generic/platform=iOS",
-              test-destination: "platform=iOS Simulator,OS=${{ inputs.ios-version }},name=iPhone 16",
+              test-destination: "platform=iOS Simulator,OS=${{ inputs.ios-version }},name=${{ inputs.ios-device }}",
             }
           - {
               name: "watchOS",
@@ -46,7 +52,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: maxim-lobanov/setup-xcode@v1
         with:
-          xcode-version: latest-stable
+          xcode-version: ${{ inputs.xcode-version || 'latest-stable' }}
       - name: Setup environment
         run: make setup
       - name: Setup Cache
@@ -59,14 +65,17 @@ jobs:
         if: steps.model-cache.outputs.cache-hit != 'true'
         run: make download-model MODEL=tiny
       - name: Install and discover destinations
+        if: ${{ matrix.run-config['condition'] == true }}
         run: |
           if [[ "${{ matrix.run-config['name'] }}" != "macOS" ]]; then
             xcodebuild -downloadPlatform ${{ matrix.run-config['name'] }}
           fi
+          echo "Runtimes for testing:"
+          xcrun simctl list runtimes
           echo "Destinations for testing:"
           xcodebuild test-without-building -only-testing WhisperKitTests/UnitTests -scheme whisperkit-Package -showdestinations
       - name: Boot Simulator and Wait
-        if: ${{ matrix.run-config['name'] != 'macOS' }} && ${{ inputs.macos-runner == 'macos-15' }}
+        if: ${{ matrix.run-config['condition'] == true }} && ${{ matrix.run-config['name'] != 'macOS' }} && ${{ inputs.macos-runner == 'macos-15' }}
         # Slower runners require some time to fully boot the simulator
         # Parse the simulator name from the destination string, boot it, and wait
         run: |
@@ -75,19 +84,16 @@ jobs:
           sleep 15
           xcrun simctl list devices
       - name: Build and Test - ${{ matrix.run-config['name'] }}
-        id: test-step
         if: ${{ matrix.run-config['condition'] == true }}
-        continue-on-error: true
         run: |
           set -o pipefail
           xcodebuild clean build-for-testing -scheme whisperkit-Package -destination '${{ matrix.run-config['clean-destination'] }}' | xcpretty
           xcodebuild test -only-testing WhisperKitTests/UnitTests -scheme whisperkit-Package -destination '${{ matrix.run-config['test-destination'] }}'
-
       - name: Upload Test Results
-        if: failure() && steps.test-step.outcome == 'failure'
+        if: failure()
         uses: actions/upload-artifact@v4
         with:
-          name: test-results-${{ matrix.run-config['name'] }}
+          name: test-results-${{ matrix.run-config['name']}}-on-${{ inputs.macos-runner }}
           path: |
             ~/Library/Developer/Xcode/DerivedData/**/Logs/Test/*.xcresult
           retention-days: 5