diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09ee495af..9a1d6e7d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -198,3 +198,15 @@ jobs: uses: ./.github/workflows/integration-test-windows.yml with: unreal-version: ${{ matrix.unreal }} + + integration-test-android: + needs: [test-android] + name: Android UE ${{ matrix.unreal }} + secrets: inherit + strategy: + fail-fast: false + matrix: + unreal: ['5.4', '5.5', '5.6', '5.7'] + uses: ./.github/workflows/integration-test-android.yml + with: + unreal-version: ${{ matrix.unreal }} diff --git a/.github/workflows/integration-test-android.yml b/.github/workflows/integration-test-android.yml new file mode 100644 index 000000000..97cb86769 --- /dev/null +++ b/.github/workflows/integration-test-android.yml @@ -0,0 +1,55 @@ +on: + workflow_call: + inputs: + unreal-version: + required: true + type: string + +jobs: + integration-test: + name: Integration Test + runs-on: ubuntu-latest + + env: + SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} + SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} + GITHUB_TOKEN: ${{ github.token }} + SAUCE_REGION: eu-central-1 + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Download sample build + uses: actions/download-artifact@v4 + with: + name: UE ${{ inputs.unreal-version }} sample build (Android) + path: sample-build + + - name: Install Pester + shell: pwsh + run: Install-Module -Name Pester -Force -SkipPublisherCheck + + - name: Run integration tests + id: run-integration-tests + shell: pwsh + env: + SENTRY_UNREAL_TEST_DSN: ${{ secrets.SENTRY_UNREAL_TEST_DSN }} + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_API_TOKEN }} + SENTRY_UNREAL_TEST_APP_PATH: ${{ github.workspace }}/sample-build/SentryPlayground-arm64.apk + UNREAL_VERSION: ${{ inputs.unreal-version }} + run: | + cd integration-test + mkdir build + cmake -B build -S . + Invoke-Pester Integration.Tests.Android.SauceLabs.ps1 -CI + + - name: Upload integration test output + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: UE ${{ inputs.unreal-version }} integration test output (Android) + path: | + integration-test/output/ + retention-days: 14 \ No newline at end of file diff --git a/integration-test/Integration.Tests.Android.Adb.ps1 b/integration-test/Integration.Tests.Android.Adb.ps1 new file mode 100644 index 000000000..08e83dd57 --- /dev/null +++ b/integration-test/Integration.Tests.Android.Adb.ps1 @@ -0,0 +1,418 @@ +# Integration tests for Sentry Unreal SDK on Android +# Requires: +# - Pre-built APK (x64 for emulator) +# - Android emulator or device connected +# - Environment variables: SENTRY_UNREAL_TEST_DSN, SENTRY_AUTH_TOKEN, SENTRY_UNREAL_TEST_APP_PATH + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function script:Get-AndroidDeviceId { + $lines = adb devices | Select-String "device$" + + if (-not $lines) { + throw "No Android devices found. Is emulator running?" + } + + # Extract device ID from the first matching line + # Line format: "emulator-5554 device" + $firstLine = $lines | Select-Object -First 1 + $deviceId = ($firstLine.Line -split '\s+')[0] + + if (-not $deviceId) { + throw "Could not extract device ID from: $($firstLine.Line)" + } + + return $deviceId +} + +function script:Install-AndroidApp { + param( + [Parameter(Mandatory)] + [string]$ApkPath, + + [Parameter(Mandatory)] + [string]$PackageName, + + [Parameter(Mandatory)] + [string]$DeviceId + ) + + if (-not (Test-Path $ApkPath)) { + throw "APK file not found: $ApkPath" + } + + if ($ApkPath -notlike '*.apk') { + throw "Package must be an .apk file. Got: $ApkPath" + } + + # Check for existing installation + Write-Debug "Checking for existing package: $PackageName" + $installed = adb -s $DeviceId shell pm list packages | Select-String -Pattern $PackageName -SimpleMatch + + if ($installed) { + Write-Host "Uninstalling previous version..." -ForegroundColor Yellow + adb -s $DeviceId uninstall $PackageName | Out-Null + Start-Sleep -Seconds 1 + } + + # Install APK + Write-Host "Installing APK to device: $DeviceId" -ForegroundColor Yellow + $installOutput = adb -s $DeviceId install -r $ApkPath 2>&1 | Out-String + + if ($LASTEXITCODE -ne 0 -or $installOutput -notmatch "Success") { + throw "Failed to install APK (exit code: $LASTEXITCODE): $installOutput" + } + + Write-Host "Package installed successfully: $PackageName" -ForegroundColor Green +} + +function script:Invoke-AndroidApp { + param( + [Parameter(Mandatory)] + [string]$ExecutablePath, + + [Parameter()] + [string]$Arguments = "" + ) + + # Extract package name from activity path (format: package.name/activity.name) + if ($ExecutablePath -notmatch '^([^/]+)/') { + throw "ExecutablePath must be in format 'package.name/activity.name'. Got: $ExecutablePath" + } + $packageName = $matches[1] + + # Use script-level Android configuration + $deviceId = $script:DeviceId + $outputDir = $script:OutputDir + + # Android-specific timeout configuration + $timeoutSeconds = 300 + $initialWaitSeconds = 3 + $pidRetrySeconds = 30 + $logPollIntervalSeconds = 2 + + $timestamp = Get-Date -Format 'yyyyMMdd-HHmmss' + $logFile = if ($OutputDir) { "$OutputDir/$timestamp-logcat.txt" } else { $null } + + # Clear logcat before launch + Write-Debug "Clearing logcat on device: $deviceId" + adb -s $deviceId logcat -c + + # Launch activity with Intent extras + Write-Host "Launching: $ExecutablePath" -ForegroundColor Cyan + if ($Arguments) { + Write-Host " Arguments: $Arguments" -ForegroundColor Cyan + } + + $startOutput = adb -s $deviceId shell am start -n $ExecutablePath $Arguments -W 2>&1 | Out-String + + if ($startOutput -match "Error") { + throw "Failed to start activity: $startOutput" + } + + # Get process ID (with retries) + Write-Debug "Waiting for app process..." + Start-Sleep -Seconds $initialWaitSeconds + + $appPID = $null + for ($i = 0; $i -lt $pidRetrySeconds; $i++) { + $pidOutput = adb -s $deviceId shell pidof $packageName 2>&1 + if ($pidOutput) { + $pidOutput = $pidOutput.ToString().Trim() + if ($pidOutput -match '^\d+$') { + $appPID = $pidOutput + break + } + } + Start-Sleep -Seconds 1 + } + + # Initialize log cache as array for consistent type handling + [array]$logCache = @() + + if (-not $appPID) { + # App might have already exited (fast message test) - capture logs anyway + Write-Host "Warning: Could not find process ID (app may have exited quickly)" -ForegroundColor Yellow + $logCache = @(adb -s $deviceId logcat -d 2>&1) + $exitCode = 0 + } else { + Write-Host "App PID: $appPID" -ForegroundColor Green + + # Monitor logcat for test completion + $startTime = Get-Date + $completed = $false + + while ((Get-Date) - $startTime -lt [TimeSpan]::FromSeconds($timeoutSeconds)) { + $newLogs = adb -s $deviceId logcat -d --pid=$appPID 2>&1 + if ($newLogs) { + $logCache = @($newLogs) + + # Check for completion markers from SentryPlaygroundGameInstance + if (($newLogs | Select-String "TEST_RESULT:") -or + ($newLogs | Select-String "Requesting app exit")) { + $completed = $true + Write-Host "Test completion detected" -ForegroundColor Green + break + } + } + + Start-Sleep -Seconds $logPollIntervalSeconds + } + + if (-not $completed) { + Write-Host "Warning: Test did not complete within timeout" -ForegroundColor Yellow + } + + $exitCode = 0 # Android apps don't report exit codes via adb + } + + # Save logcat to file if OutputDir specified + if ($logFile) { + $logCache | Out-File $logFile + Write-Host "Logcat saved to: $logFile" + } + + # Return structured result (matches app-runner pattern) + return @{ + ExitCode = $exitCode + Output = $logCache + Error = @() + } +} + +BeforeAll { + # Check if configuration file exists + $configFile = "$PSScriptRoot/TestConfig.local.ps1" + if (-not (Test-Path $configFile)) { + throw "Configuration file '$configFile' not found. Run 'cmake -B build -S .' first" + } + + # Load configuration (provides $global:AppRunnerPath) + . $configFile + + # Import app-runner modules (SentryApiClient, test utilities) + . "$global:AppRunnerPath/import-modules.ps1" + + # Validate environment variables + $script:DSN = $env:SENTRY_UNREAL_TEST_DSN + $script:AuthToken = $env:SENTRY_AUTH_TOKEN + $script:ApkPath = $env:SENTRY_UNREAL_TEST_APP_PATH + + if (-not $script:DSN) { + throw "Environment variable SENTRY_UNREAL_TEST_DSN must be set" + } + + if (-not $script:AuthToken) { + throw "Environment variable SENTRY_AUTH_TOKEN must be set" + } + + if (-not $script:ApkPath) { + throw "Environment variable SENTRY_UNREAL_TEST_APP_PATH must be set" + } + + # Connect to Sentry API + Write-Host "Connecting to Sentry API..." -ForegroundColor Yellow + Connect-SentryApi -DSN $script:DSN -ApiToken $script:AuthToken + + # Validate app path + if (-not (Test-Path $script:ApkPath)) { + throw "Application not found at: $script:ApkPath" + } + + # Create output directory + $script:OutputDir = "$PSScriptRoot/output" + if (-not (Test-Path $script:OutputDir)) { + New-Item -ItemType Directory -Path $script:OutputDir | Out-Null + } + + $script:PackageName = "io.sentry.unreal.sample" + $script:ActivityName = "$script:PackageName/com.epicgames.unreal.GameActivity" + + # Get Android device + $script:DeviceId = Get-AndroidDeviceId + Write-Host "Found Android device: $script:DeviceId" -ForegroundColor Green + + # Install APK to device + Write-Host "Installing APK to Android device..." -ForegroundColor Yellow + Install-AndroidApp -ApkPath $script:ApkPath -PackageName $script:PackageName -DeviceId $script:DeviceId + + # ========================================== + # RUN 1: Crash test - creates minidump + # ========================================== + # The crash is captured but NOT uploaded yet (Android behavior). + # TODO: Re-enable once Android SDK tag persistence is fixed (`test.crash_id` tag set before crash is not synced to the captured crash on Android) + + # Write-Host "Running crash-capture test (will crash)..." -ForegroundColor Yellow + # $cmdlineCrashArgs = "-e cmdline '-crash-capture'" + # $global:AndroidCrashResult = Invoke-AndroidApp -ExecutablePath $script:ActivityName -Arguments $cmdlineCrashArgs + + # Write-Host "Crash test exit code: $($global:AndroidCrashResult.ExitCode)" -ForegroundColor Cyan + + # ========================================== + # RUN 2: Message test - uploads crash from Run 1 + captures message + # ========================================== + # Currently we need to run again so that Sentry sends the crash event captured during the previous app session. + # TODO: use -SkipReinstall to preserve the crash state. + + Write-Host "Running message-capture test (will upload crash from previous run)..." -ForegroundColor Yellow + $cmdlineMessageArgs = "-e cmdline '-message-capture'" + $global:AndroidMessageResult = Invoke-AndroidApp -ExecutablePath $script:ActivityName -Arguments $cmdlineMessageArgs + + Write-Host "Message test exit code: $($global:AndroidMessageResult.ExitCode)" -ForegroundColor Cyan +} + +Describe "Sentry Unreal Android Integration Tests" { + + # ========================================== + # NOTE: Crash Capture Tests are DISABLED due to tag sync issue + # Uncomment when Android SDK tag persistence is fixed + # ========================================== + # Context "Crash Capture Tests" { + # BeforeAll { + # # Crash event is sent during the MESSAGE run (Run 2) + # # But the crash_id comes from the CRASH run (Run 1) + # $CrashResult = $global:AndroidCrashResult + # $CrashEvent = $null + # + # # Parse crash event ID from crash run output + # $eventIds = Get-EventIds -AppOutput $CrashResult.Output -ExpectedCount 1 + # + # if ($eventIds -and $eventIds.Count -gt 0) { + # Write-Host "Crash ID captured: $($eventIds[0])" -ForegroundColor Cyan + # $crashId = $eventIds[0] + # + # # Fetch crash event using the tag (event was sent during message run) + # try { + # $CrashEvent = Get-SentryTestEvent -TagName 'test.crash_id' -TagValue "$crashId" + # Write-Host "Crash event fetched from Sentry successfully" -ForegroundColor Green + # } catch { + # Write-Host "Failed to fetch crash event from Sentry: $_" -ForegroundColor Red + # } + # } else { + # Write-Host "Warning: No crash event ID found in output" -ForegroundColor Yellow + # } + # } + # + # It "Should output event ID before crash" { + # $eventIds = Get-EventIds -AppOutput $CrashResult.Output -ExpectedCount 1 + # $eventIds | Should -Not -BeNullOrEmpty + # $eventIds.Count | Should -Be 1 + # } + # + # It "Should capture crash event in Sentry (uploaded during next run)" { + # $CrashEvent | Should -Not -BeNullOrEmpty + # } + # + # It "Should have correct event type and platform" { + # $CrashEvent.type | Should -Be 'error' + # $CrashEvent.platform | Should -Be 'native' + # } + # + # It "Should have exception information" { + # $CrashEvent.exception | Should -Not -BeNullOrEmpty + # $CrashEvent.exception.values | Should -Not -BeNullOrEmpty + # } + # + # It "Should have stack trace" { + # $exception = $CrashEvent.exception.values[0] + # $exception.stacktrace | Should -Not -BeNullOrEmpty + # $exception.stacktrace.frames | Should -Not -BeNullOrEmpty + # } + # + # It "Should have user context" { + # $CrashEvent.user | Should -Not -BeNullOrEmpty + # $CrashEvent.user.username | Should -Be 'TestUser' + # $CrashEvent.user.email | Should -Be 'user-mail@test.abc' + # $CrashEvent.user.id | Should -Be '12345' + # } + # + # It "Should have test.crash_id tag for correlation" { + # $tags = $CrashEvent.tags + # $crashIdTag = $tags | Where-Object { $_.key -eq 'test.crash_id' } + # $crashIdTag | Should -Not -BeNullOrEmpty + # $crashIdTag.value | Should -Not -BeNullOrEmpty + # } + # + # It "Should have integration test tag" { + # $tags = $CrashEvent.tags + # ($tags | Where-Object { $_.key -eq 'test.suite' }).value | Should -Be 'integration' + # } + # + # It "Should have breadcrumbs from before crash" { + # $CrashEvent.breadcrumbs | Should -Not -BeNullOrEmpty + # $CrashEvent.breadcrumbs.values | Should -Not -BeNullOrEmpty + # } + # } + + Context "Message Capture Tests" { + BeforeAll { + $MessageResult = $global:AndroidMessageResult + $MessageEvent = $null + + # Parse event ID from output + $eventIds = Get-EventIds -AppOutput $MessageResult.Output -ExpectedCount 1 + + if ($eventIds -and $eventIds.Count -gt 0) { + Write-Host "Message event ID captured: $($eventIds[0])" -ForegroundColor Cyan + + # Fetch event from Sentry (with polling) + try { + $MessageEvent = Get-SentryTestEvent -EventId $eventIds[0] + Write-Host "Message event fetched from Sentry successfully" -ForegroundColor Green + } catch { + Write-Host "Failed to fetch message event from Sentry: $_" -ForegroundColor Red + } + } else { + Write-Host "Warning: No message event ID found in output" -ForegroundColor Yellow + } + } + + It "Should output event ID" { + $eventIds = Get-EventIds -AppOutput $MessageResult.Output -ExpectedCount 1 + $eventIds | Should -Not -BeNullOrEmpty + $eventIds.Count | Should -Be 1 + } + + It "Should output TEST_RESULT with success" { + $testResultLine = $MessageResult.Output | Where-Object { $_ -match 'TEST_RESULT:' } + $testResultLine | Should -Not -BeNullOrEmpty + $testResultLine | Should -Match '"success"\s*:\s*true' + } + + It "Should capture message event in Sentry" { + $MessageEvent | Should -Not -BeNullOrEmpty + } + + It "Should have correct platform" { + # Android events are captured from Java layer, so platform is 'java' not 'native' + $MessageEvent.platform | Should -Be 'java' + } + + It "Should have message content" { + $MessageEvent.message | Should -Not -BeNullOrEmpty + $MessageEvent.message.formatted | Should -Match 'Integration test message' + } + + It "Should have user context" { + $MessageEvent.user | Should -Not -BeNullOrEmpty + $MessageEvent.user.username | Should -Be 'TestUser' + } + + It "Should have integration test tag" { + $tags = $MessageEvent.tags + ($tags | Where-Object { $_.key -eq 'test.suite' }).value | Should -Be 'integration' + } + + It "Should have breadcrumbs" { + $MessageEvent.breadcrumbs | Should -Not -BeNullOrEmpty + $MessageEvent.breadcrumbs.values | Should -Not -BeNullOrEmpty + } + } +} + +AfterAll { + Write-Host "Disconnecting from Sentry API..." -ForegroundColor Yellow + Disconnect-SentryApi + Write-Host "Integration tests complete" -ForegroundColor Green +} \ No newline at end of file diff --git a/integration-test/Integration.Tests.Android.SauceLabs.ps1 b/integration-test/Integration.Tests.Android.SauceLabs.ps1 new file mode 100644 index 000000000..6ec5e115b --- /dev/null +++ b/integration-test/Integration.Tests.Android.SauceLabs.ps1 @@ -0,0 +1,604 @@ +# Integration tests for Sentry Unreal SDK on Android via SauceLabs Real Device Cloud +# Requires: +# - Pre-built APK +# - SauceLabs account credentials +# - Environment variables: SENTRY_UNREAL_TEST_DSN, SENTRY_AUTH_TOKEN, SENTRY_UNREAL_TEST_APP_PATH +# SAUCE_USERNAME, SAUCE_ACCESS_KEY, SAUCE_REGION + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# Script-level state for session management +$script:SessionId = $null +$script:LogLineCount = 0 # Track log lines read so far (for delta) + +function script:Invoke-SauceLabsApi { + param( + [Parameter(Mandatory)] + [string]$Method, + + [Parameter(Mandatory)] + [string]$Uri, + + [Parameter()] + [hashtable]$Body = $null, + + [Parameter()] + [string]$ContentType = 'application/json', + + [Parameter()] + [switch]$IsMultipart, + + [Parameter()] + [string]$FilePath + ) + + $username = $env:SAUCE_USERNAME + $accessKey = $env:SAUCE_ACCESS_KEY + + if (-not $username -or -not $accessKey) { + throw "SAUCE_USERNAME and SAUCE_ACCESS_KEY environment variables must be set" + } + + $base64Auth = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes("${username}:${accessKey}")) + $headers = @{ + 'Authorization' = "Basic $base64Auth" + } + + try { + if ($IsMultipart) { + # Use curl for multipart uploads (PowerShell's Invoke-WebRequest struggles with this) + $curlCmd = "curl -u `"$username`:$accessKey`" -X $Method `"$Uri`" -F `"payload=@$FilePath`" -F `"name=$(Split-Path $FilePath -Leaf)`"" + Write-Debug "Executing: $curlCmd" + $response = Invoke-Expression $curlCmd | ConvertFrom-Json + return $response + } else { + $params = @{ + Uri = $Uri + Method = $Method + Headers = $headers + } + + if ($Body) { + $params['Body'] = ($Body | ConvertTo-Json -Depth 10) + $params['ContentType'] = $ContentType + } + + $response = Invoke-RestMethod @params + return $response + } + } catch { + Write-Error "SauceLabs API call failed: $($_.Exception.Message)" + if ($_.Exception.Response) { + $reader = [System.IO.StreamReader]::new($_.Exception.Response.GetResponseStream()) + $responseBody = $reader.ReadToEnd() + Write-Error "Response: $responseBody" + } + throw + } +} + +function script:Install-SauceLabsApp { + param( + [Parameter(Mandatory)] + [string]$ApkPath, + + [Parameter(Mandatory)] + [string]$Region + ) + + if (-not (Test-Path $ApkPath)) { + throw "APK file not found: $ApkPath" + } + + if ($ApkPath -notlike '*.apk') { + throw "Package must be an .apk file. Got: $ApkPath" + } + + Write-Host "Uploading APK to SauceLabs Storage..." -ForegroundColor Yellow + $uploadUri = "https://api.${Region}.saucelabs.com/v1/storage/upload" + + $response = Invoke-SauceLabsApi -Method POST -Uri $uploadUri -IsMultipart -FilePath $ApkPath + + if (-not $response.item.id) { + throw "Failed to upload APK: No storage ID in response" + } + + $storageId = $response.item.id + Write-Host "APK uploaded successfully. Storage ID: $storageId" -ForegroundColor Green + + return $storageId +} + +function script:Initialize-SauceLabsSession { + param( + [Parameter(Mandatory)] + [string]$StorageId, + + [Parameter(Mandatory)] + [string]$DeviceName, + + [Parameter(Mandatory)] + [string]$Region, + + [Parameter(Mandatory)] + [string]$UnrealVersion + ) + + Write-Host "Creating SauceLabs Appium session..." -ForegroundColor Yellow + + $sessionUri = "https://ondemand.${Region}.saucelabs.com/wd/hub/session" + + $capabilities = @{ + capabilities = @{ + alwaysMatch = @{ + platformName = "Android" + 'appium:app' = "storage:$StorageId" + 'appium:deviceName' = $DeviceName + 'appium:automationName' = "UiAutomator2" + 'appium:noReset' = $true + 'appium:autoLaunch' = $false + 'sauce:options' = @{ + name = "$UnrealVersion Android Integration Test" + appiumVersion = "latest" + } + } + } + } + + $response = Invoke-SauceLabsApi -Method POST -Uri $sessionUri -Body $capabilities + + $sessionId = $response.value.sessionId + if (-not $sessionId) { + $sessionId = $response.sessionId + } + + if (-not $sessionId) { + throw "Failed to create session: No session ID in response" + } + + Write-Host "Session created successfully. Session ID: $sessionId" -ForegroundColor Green + + return $sessionId +} + +function script:Invoke-SauceLabsApp { + param( + [Parameter(Mandatory)] + [string]$PackageName, + + [Parameter(Mandatory)] + [string]$ActivityName, + + [Parameter()] + [string]$Arguments = "", + + [Parameter(Mandatory)] + [string]$Region + ) + + $sessionId = $script:SessionId + if (-not $sessionId) { + throw "No active SauceLabs session. Call Initialize-SauceLabsSession first." + } + + $baseUri = "https://ondemand.${Region}.saucelabs.com/wd/hub/session/$sessionId" + $outputDir = $script:OutputDir + + # Configuration + $timeoutSeconds = 300 + $pollIntervalSeconds = 2 + + $timestamp = Get-Date -Format 'yyyyMMdd-HHmmss' + $logFile = if ($OutputDir) { "$OutputDir/$timestamp-logcat.txt" } else { $null } + + # Launch activity with Intent extras + Write-Host "Launching: $PackageName/$ActivityName" -ForegroundColor Cyan + if ($Arguments) { + Write-Host " Arguments: $Arguments" -ForegroundColor Cyan + } + + $launchBody = @{ + appPackage = $PackageName + appActivity = $ActivityName + appWaitActivity = "*" + intentAction = "android.intent.action.MAIN" + intentCategory = "android.intent.category.LAUNCHER" + } + + if ($Arguments) { + $launchBody['optionalIntentArguments'] = $Arguments + } + + try { + $launchResponse = Invoke-SauceLabsApi -Method POST -Uri "$baseUri/appium/device/start_activity" -Body $launchBody + Write-Debug "Launch response: $($launchResponse | ConvertTo-Json)" + } catch { + throw "Failed to launch activity: $_" + } + + # Wait a moment for app to start + Start-Sleep -Seconds 3 + + # Poll app state until it exits or completes + Write-Host "Monitoring app execution..." -ForegroundColor Yellow + $startTime = Get-Date + $completed = $false + + while ((Get-Date) - $startTime -lt [TimeSpan]::FromSeconds($timeoutSeconds)) { + # Query app state + $stateBody = @{ + script = "mobile: queryAppState" + args = @( + @{ appId = $PackageName } + ) + } + + try { + $stateResponse = Invoke-SauceLabsApi -Method POST -Uri "$baseUri/execute/sync" -Body $stateBody + $appState = $stateResponse.value + + Write-Debug "App state: $appState (elapsed: $([int]((Get-Date) - $startTime).TotalSeconds)s)" + + # State 1 = not running, 0 = not installed + if ($appState -eq 1 -or $appState -eq 0) { + Write-Host "App finished/crashed (state: $appState)" -ForegroundColor Green + $completed = $true + break + } + + # Also check logs for completion markers + $logBody = @{ type = "logcat" } + $logResponse = Invoke-SauceLabsApi -Method POST -Uri "$baseUri/log" -Body $logBody + + if ($logResponse.value -and $logResponse.value.Count -gt $script:LogLineCount) { + # Use array slicing instead of Select-Object -Skip (more reliable) + $newLogs = @($logResponse.value[$script:LogLineCount..($logResponse.value.Count - 1)]) + $logMessages = @($newLogs | ForEach-Object { if ($_) { $_.message } }) + + # Check for completion markers + if (($logMessages | Where-Object { $_ -match "TEST_RESULT:" }) -or + ($logMessages | Where-Object { $_ -match "Requesting app exit" })) { + Write-Host "Test completion detected in logs" -ForegroundColor Green + $completed = $true + break + } + } + } catch { + Write-Warning "Failed to query app state: $_" + } + + Start-Sleep -Seconds $pollIntervalSeconds + } + + if (-not $completed) { + Write-Host "Warning: Test did not complete within timeout" -ForegroundColor Yellow + } + + # Retrieve final logs (delta from last read) + Write-Host "Retrieving logs..." -ForegroundColor Yellow + $logBody = @{ type = "logcat" } + $logResponse = Invoke-SauceLabsApi -Method POST -Uri "$baseUri/log" -Body $logBody + + # Extract new log lines (delta) + [array]$allLogs = @() + if ($logResponse.value -and $logResponse.value.Count -gt 0) { + $totalLogCount = $logResponse.value.Count + Write-Debug "Total logs in response: $totalLogCount, Previously read: $script:LogLineCount" + + if ($totalLogCount -gt $script:LogLineCount) { + # Get only new logs using array slicing (more reliable than Select-Object -Skip) + $allLogs = @($logResponse.value[$script:LogLineCount..($totalLogCount - 1)]) + Write-Host "Retrieved $($allLogs.Count) new log lines" -ForegroundColor Cyan + } else { + Write-Host "No new log lines since last read" -ForegroundColor Yellow + } + + # Update counter for next read + $script:LogLineCount = $totalLogCount + } else { + Write-Host "No logs available in response" -ForegroundColor Yellow + } + + # Convert SauceLabs log format to text (matching adb output) + $logCache = @() + if ($allLogs -and $allLogs.Count -gt 0) { + $logCache = $allLogs | ForEach-Object { + if ($_) { + $timestamp = if ($_.timestamp) { $_.timestamp } else { "" } + $level = if ($_.level) { $_.level } else { "" } + $message = if ($_.message) { $_.message } else { "" } + "$timestamp $level $message" + } + } | Where-Object { $_ } # Filter out any nulls + } + + # Save logs to file if OutputDir specified + if ($logFile) { + $logCache | Out-File $logFile + Write-Host "Logs saved to: $logFile" -ForegroundColor Cyan + } + + # Return structured result (matches app-runner pattern) + return @{ + ExitCode = 0 # Android apps don't report exit codes + Output = $logCache + Error = @() + } +} + +BeforeAll { + # Check if configuration file exists + $configFile = "$PSScriptRoot/TestConfig.local.ps1" + if (-not (Test-Path $configFile)) { + throw "Configuration file '$configFile' not found. Run 'cmake -B build -S .' first" + } + + # Load configuration (provides $global:AppRunnerPath) + . $configFile + + # Import app-runner modules (SentryApiClient, test utilities) + . "$global:AppRunnerPath/import-modules.ps1" + + # Validate environment variables + $script:DSN = $env:SENTRY_UNREAL_TEST_DSN + $script:AuthToken = $env:SENTRY_AUTH_TOKEN + $script:ApkPath = $env:SENTRY_UNREAL_TEST_APP_PATH + $script:SauceUsername = $env:SAUCE_USERNAME + $script:SauceAccessKey = $env:SAUCE_ACCESS_KEY + $script:SauceRegion = $env:SAUCE_REGION + + if (-not $script:DSN) { + throw "Environment variable SENTRY_UNREAL_TEST_DSN must be set" + } + + if (-not $script:AuthToken) { + throw "Environment variable SENTRY_AUTH_TOKEN must be set" + } + + if (-not $script:ApkPath) { + throw "Environment variable SENTRY_UNREAL_TEST_APP_PATH must be set" + } + + if (-not $script:SauceUsername) { + throw "Environment variable SAUCE_USERNAME must be set" + } + + if (-not $script:SauceAccessKey) { + throw "Environment variable SAUCE_ACCESS_KEY must be set" + } + + if (-not $script:SauceRegion) { + throw "Environment variable SAUCE_REGION must be set" + } + + # Connect to Sentry API + Write-Host "Connecting to Sentry API..." -ForegroundColor Yellow + Connect-SentryApi -DSN $script:DSN -ApiToken $script:AuthToken + + # Validate app path + if (-not (Test-Path $script:ApkPath)) { + throw "Application not found at: $script:ApkPath" + } + + # Create output directory + $script:OutputDir = "$PSScriptRoot/output" + if (-not (Test-Path $script:OutputDir)) { + New-Item -ItemType Directory -Path $script:OutputDir | Out-Null + } + + $script:PackageName = "io.sentry.unreal.sample" + $script:ActivityName = "com.epicgames.unreal.GameActivity" + $script:DeviceName = "Samsung_Galaxy_S23_FE_free" + $script:UnrealVersion = if ($env:UNREAL_VERSION) { $env:UNREAL_VERSION } else { "5.x" } + + # Upload APK to SauceLabs Storage + Write-Host "Uploading APK to SauceLabs..." -ForegroundColor Yellow + $script:StorageId = Install-SauceLabsApp -ApkPath $script:ApkPath -Region $script:SauceRegion + + # Create SauceLabs session (reused for all app launches) + Write-Host "Creating SauceLabs session..." -ForegroundColor Yellow + $script:SessionId = Initialize-SauceLabsSession ` + -StorageId $script:StorageId ` + -DeviceName $script:DeviceName ` + -Region $script:SauceRegion ` + -UnrealVersion $script:UnrealVersion + + # ========================================== + # RUN 1: Crash test - creates minidump + # ========================================== + # The crash is captured but NOT uploaded yet (Android behavior). + # TODO: Re-enable once Android SDK tag persistence is fixed (`test.crash_id` tag set before crash is not synced to the captured crash on Android) + + # Write-Host "Running crash-capture test (will crash)..." -ForegroundColor Yellow + # $cmdlineCrashArgs = "-e cmdline '-crash-capture'" + # $global:SauceLabsCrashResult = Invoke-SauceLabsApp ` + # -PackageName $script:PackageName ` + # -ActivityName $script:ActivityName ` + # -Arguments $cmdlineCrashArgs ` + # -Region $script:SauceRegion + + # Write-Host "Crash test exit code: $($global:SauceLabsCrashResult.ExitCode)" -ForegroundColor Cyan + + # ========================================== + # RUN 2: Message test - uploads crash from Run 1 + captures message + # ========================================== + # Currently we need to run again so that Sentry sends the crash event captured during the previous app session. + + Write-Host "Running message-capture test (will upload crash from previous run)..." -ForegroundColor Yellow + $cmdlineMessageArgs = "-e cmdline '-message-capture'" + $global:SauceLabsMessageResult = Invoke-SauceLabsApp ` + -PackageName $script:PackageName ` + -ActivityName $script:ActivityName ` + -Arguments $cmdlineMessageArgs ` + -Region $script:SauceRegion + + Write-Host "Message test exit code: $($global:SauceLabsMessageResult.ExitCode)" -ForegroundColor Cyan +} + +Describe "Sentry Unreal Android Integration Tests (SauceLabs)" { + + # ========================================== + # NOTE: Crash Capture Tests are DISABLED due to tag sync issue + # Uncomment when Android SDK tag persistence is fixed + # ========================================== + # Context "Crash Capture Tests" { + # BeforeAll { + # # Crash event is sent during the MESSAGE run (Run 2) + # # But the crash_id comes from the CRASH run (Run 1) + # $CrashResult = $global:SauceLabsCrashResult + # $CrashEvent = $null + # + # # Parse crash event ID from crash run output + # $eventIds = Get-EventIds -AppOutput $CrashResult.Output -ExpectedCount 1 + # + # if ($eventIds -and $eventIds.Count -gt 0) { + # Write-Host "Crash ID captured: $($eventIds[0])" -ForegroundColor Cyan + # $crashId = $eventIds[0] + # + # # Fetch crash event using the tag (event was sent during message run) + # try { + # $CrashEvent = Get-SentryTestEvent -TagName 'test.crash_id' -TagValue "$crashId" + # Write-Host "Crash event fetched from Sentry successfully" -ForegroundColor Green + # } catch { + # Write-Host "Failed to fetch crash event from Sentry: $_" -ForegroundColor Red + # } + # } else { + # Write-Host "Warning: No crash event ID found in output" -ForegroundColor Yellow + # } + # } + # + # It "Should output event ID before crash" { + # $eventIds = Get-EventIds -AppOutput $CrashResult.Output -ExpectedCount 1 + # $eventIds | Should -Not -BeNullOrEmpty + # $eventIds.Count | Should -Be 1 + # } + # + # It "Should capture crash event in Sentry (uploaded during next run)" { + # $CrashEvent | Should -Not -BeNullOrEmpty + # } + # + # It "Should have correct event type and platform" { + # $CrashEvent.type | Should -Be 'error' + # $CrashEvent.platform | Should -Be 'native' + # } + # + # It "Should have exception information" { + # $CrashEvent.exception | Should -Not -BeNullOrEmpty + # $CrashEvent.exception.values | Should -Not -BeNullOrEmpty + # } + # + # It "Should have stack trace" { + # $exception = $CrashEvent.exception.values[0] + # $exception.stacktrace | Should -Not -BeNullOrEmpty + # $exception.stacktrace.frames | Should -Not -BeNullOrEmpty + # } + # + # It "Should have user context" { + # $CrashEvent.user | Should -Not -BeNullOrEmpty + # $CrashEvent.user.username | Should -Be 'TestUser' + # $CrashEvent.user.email | Should -Be 'user-mail@test.abc' + # $CrashEvent.user.id | Should -Be '12345' + # } + # + # It "Should have test.crash_id tag for correlation" { + # $tags = $CrashEvent.tags + # $crashIdTag = $tags | Where-Object { $_.key -eq 'test.crash_id' } + # $crashIdTag | Should -Not -BeNullOrEmpty + # $crashIdTag.value | Should -Not -BeNullOrEmpty + # } + # + # It "Should have integration test tag" { + # $tags = $CrashEvent.tags + # ($tags | Where-Object { $_.key -eq 'test.suite' }).value | Should -Be 'integration' + # } + # + # It "Should have breadcrumbs from before crash" { + # $CrashEvent.breadcrumbs | Should -Not -BeNullOrEmpty + # $CrashEvent.breadcrumbs.values | Should -Not -BeNullOrEmpty + # } + # } + + Context "Message Capture Tests" { + BeforeAll { + $MessageResult = $global:SauceLabsMessageResult + $MessageEvent = $null + + # Parse event ID from output + $eventIds = Get-EventIds -AppOutput $MessageResult.Output -ExpectedCount 1 + + if ($eventIds -and $eventIds.Count -gt 0) { + Write-Host "Message event ID captured: $($eventIds[0])" -ForegroundColor Cyan + + # Fetch event from Sentry (with polling) + try { + $MessageEvent = Get-SentryTestEvent -EventId $eventIds[0] + Write-Host "Message event fetched from Sentry successfully" -ForegroundColor Green + } catch { + Write-Host "Failed to fetch message event from Sentry: $_" -ForegroundColor Red + } + } else { + Write-Host "Warning: No message event ID found in output" -ForegroundColor Yellow + } + } + + It "Should output event ID" { + $eventIds = Get-EventIds -AppOutput $MessageResult.Output -ExpectedCount 1 + $eventIds | Should -Not -BeNullOrEmpty + $eventIds.Count | Should -Be 1 + } + + It "Should output TEST_RESULT with success" { + $testResultLine = $MessageResult.Output | Where-Object { $_ -match 'TEST_RESULT:' } + $testResultLine | Should -Not -BeNullOrEmpty + $testResultLine | Should -Match '"success"\s*:\s*true' + } + + It "Should capture message event in Sentry" { + $MessageEvent | Should -Not -BeNullOrEmpty + } + + It "Should have correct platform" { + # Android events are captured from Java layer, so platform is 'java' not 'native' + $MessageEvent.platform | Should -Be 'java' + } + + It "Should have message content" { + $MessageEvent.message | Should -Not -BeNullOrEmpty + $MessageEvent.message.formatted | Should -Match 'Integration test message' + } + + It "Should have user context" { + $MessageEvent.user | Should -Not -BeNullOrEmpty + $MessageEvent.user.username | Should -Be 'TestUser' + } + + It "Should have integration test tag" { + $tags = $MessageEvent.tags + ($tags | Where-Object { $_.key -eq 'test.suite' }).value | Should -Be 'integration' + } + + It "Should have breadcrumbs" { + $MessageEvent.breadcrumbs | Should -Not -BeNullOrEmpty + $MessageEvent.breadcrumbs.values | Should -Not -BeNullOrEmpty + } + } +} + +AfterAll { + # Clean up SauceLabs session + if ($script:SessionId) { + Write-Host "Ending SauceLabs session..." -ForegroundColor Yellow + try { + $sessionUri = "https://ondemand.$($script:SauceRegion).saucelabs.com/wd/hub/session/$($script:SessionId)" + Invoke-SauceLabsApi -Method DELETE -Uri $sessionUri + Write-Host "Session ended successfully" -ForegroundColor Green + } catch { + Write-Warning "Failed to end session: $_" + } + } + + Write-Host "Disconnecting from Sentry API..." -ForegroundColor Yellow + Disconnect-SentryApi + Write-Host "Integration tests complete" -ForegroundColor Green +} diff --git a/integration-test/README.md b/integration-test/README.md index 589f88f91..53b855d85 100644 --- a/integration-test/README.md +++ b/integration-test/README.md @@ -2,8 +2,15 @@ This directory contains integration tests for the Sentry Unreal SDK using Pester (PowerShell testing framework). +Supports testing on: +- **Windows** - Desktop (x64) +- **Linux** - Desktop (x64) +- **Android** - Local device/emulator (via adb) or SauceLabs Real Device Cloud + ## Prerequisites +### Common Requirements + - **PowerShell 7+** (Core edition) - **CMake 3.20+** - **Pester 5+** - Install with: `Install-Module -Name Pester -Force -SkipPublisherCheck` @@ -11,7 +18,20 @@ This directory contains integration tests for the Sentry Unreal SDK using Pester - **Environment variables**: - `SENTRY_UNREAL_TEST_DSN` - Sentry test project DSN - `SENTRY_AUTH_TOKEN` - Sentry API authentication token - - `SENTRY_UNREAL_TEST_APP_PATH` - Path to the SentryPlayground executable + - `SENTRY_UNREAL_TEST_APP_PATH` - Path to the SentryPlayground executable/APK + +### Android-Specific Requirements + +#### Option A: Local Testing (via adb) +- **Android device or emulator** connected and visible via `adb devices` +- **ADB (Android Debug Bridge)** installed and in PATH + +#### Option B: Cloud Testing (via SauceLabs) +- **SauceLabs account** with Real Device Cloud access +- **Additional environment variables**: + - `SAUCE_USERNAME` - SauceLabs username + - `SAUCE_ACCESS_KEY` - SauceLabs access key + - `SAUCE_REGION` - SauceLabs region (e.g., `eu-central-1`) ## Setup @@ -39,6 +59,7 @@ This will: 3. Download the appropriate artifact: - `UE X.X sample build (Windows)` for Windows testing - `UE X.X sample build (Linux)` for Linux testing + - `UE X.X sample build (Android)` for Android testing 4. Extract to a known location #### Option B: Build Locally @@ -75,11 +96,43 @@ cd integration-test pwsh -Command "Invoke-Pester Integration.Tests.ps1" ``` +### Android (Local via adb) + +```bash +# Ensure device/emulator is connected +adb devices + +# Set environment variables +export SENTRY_UNREAL_TEST_DSN="https://key@org.ingest.sentry.io/project" +export SENTRY_AUTH_TOKEN="sntrys_your_token_here" +export SENTRY_UNREAL_TEST_APP_PATH="./path/to/SentryPlayground.apk" + +# Run tests +cd integration-test +pwsh -Command "Invoke-Pester Integration.Tests.Android.Adb.ps1" +``` + +### Android (Cloud via SauceLabs) + +```bash +# Set environment variables +export SENTRY_UNREAL_TEST_DSN="https://key@org.ingest.sentry.io/project" +export SENTRY_AUTH_TOKEN="sntrys_your_token_here" +export SENTRY_UNREAL_TEST_APP_PATH="./path/to/SentryPlayground.apk" +export SAUCE_USERNAME="your-saucelabs-username" +export SAUCE_ACCESS_KEY="your-saucelabs-access-key" +export SAUCE_REGION="eu-central-1" + +# Run tests +cd integration-test +pwsh -Command "Invoke-Pester Integration.Tests.Android.SauceLabs.ps1" +``` + ## Test Coverage The integration tests cover: -### Crash Capture Tests +### Crash Capture Tests _(Windows/Linux)_ - Application crashes with non-zero exit code - Event ID is captured from output (set via `test.crash_id` tag) - Crash event appears in Sentry @@ -89,8 +142,10 @@ The integration tests cover: - Integration test tags are set - Breadcrumbs are collected -### Message Capture Tests -- Application exits cleanly (exit code 0) +**Note**: Crash capture tests are currently disabled on Android due to a known issue with tag persistence across app sessions. + +### Message Capture Tests _(All platforms)_ +- Application exits cleanly (exit code 0 on Windows/Linux, Android doesn't report exit codes) - Event ID is captured from output - TEST_RESULT indicates success - Message event appears in Sentry @@ -99,9 +154,13 @@ The integration tests cover: - Integration test tags are set - Breadcrumbs are collected +**Note**: On Android, events are captured from the Java layer, so the platform will be `java` instead of `native`. + ## Output Test outputs are saved to `integration-test/output/`: + +### Windows/Linux - `*-crash-stdout.log` - Crash test standard output - `*-crash-stderr.log` - Crash test standard error - `*-crash-result.json` - Full crash test result @@ -110,6 +169,13 @@ Test outputs are saved to `integration-test/output/`: - `*-message-result.json` - Full message test result - `event-*.json` - Events fetched from Sentry API +### Android +- `*-logcat.txt` - Logcat output from app execution (one file per launch) +- `event-*.json` - Events fetched from Sentry API + ## CI Integration -See `.github/workflows/integration-test-windows.yml` and `.github/workflows/integration-test-linux.yml` for CI usage examples. +See the following workflow files for CI usage examples: +- `.github/workflows/integration-test-windows.yml` - Windows desktop testing +- `.github/workflows/integration-test-linux.yml` - Linux desktop testing +- `.github/workflows/integration-test-android.yml` - Android testing via SauceLabs Real Device Cloud diff --git a/sample/Config/DefaultEngine.ini b/sample/Config/DefaultEngine.ini index bca84ad7e..1e8177a5d 100644 --- a/sample/Config/DefaultEngine.ini +++ b/sample/Config/DefaultEngine.ini @@ -215,6 +215,8 @@ KeyStore=debug.keystore KeyAlias=androiddebugkey KeyStorePassword=android KeyPassword=android +bBuildForArm64=True +bBuildForX8664=True [/Script/IOSRuntimeSettings.IOSRuntimeSettings] BundleIdentifier=io.sentry.unreal.sample diff --git a/sample/Source/SentryPlayground/SentryPlayground.Build.cs b/sample/Source/SentryPlayground/SentryPlayground.Build.cs index f1d2de319..35cb4df6c 100644 --- a/sample/Source/SentryPlayground/SentryPlayground.Build.cs +++ b/sample/Source/SentryPlayground/SentryPlayground.Build.cs @@ -7,14 +7,14 @@ public class SentryPlayground : ModuleRules public SentryPlayground(ReadOnlyTargetRules Target) : base(Target) { PCHUsage = PCHUsageMode.UseExplicitOrSharedPCHs; - + PublicDependencyModuleNames.AddRange(new string[] { "Core", "CoreUObject", "Engine", "InputCore", "Sentry" }); PrivateDependencyModuleNames.AddRange(new string[] { }); // Uncomment if you are using Slate UI // PrivateDependencyModuleNames.AddRange(new string[] { "Slate", "SlateCore" }); - + // Uncomment if you are using online features // PrivateDependencyModuleNames.Add("OnlineSubsystem"); diff --git a/sample/Source/SentryPlayground/SentryPlaygroundGameInstance.cpp b/sample/Source/SentryPlayground/SentryPlaygroundGameInstance.cpp index 989b52943..e5421c64e 100644 --- a/sample/Source/SentryPlayground/SentryPlaygroundGameInstance.cpp +++ b/sample/Source/SentryPlayground/SentryPlaygroundGameInstance.cpp @@ -18,14 +18,16 @@ void USentryPlaygroundGameInstance::Init() { Super::Init(); - const TCHAR* CommandLine = FCommandLine::Get(); + FString CommandLine = FCommandLine::Get(); + + UE_LOG(LogSentrySample, Display, TEXT("Startin app with commandline: %s\n"), *CommandLine); // Check for expected test parameters to decide between running integration tests // or launching the sample app with UI for manual testing - if (FParse::Param(FCommandLine::Get(), TEXT("crash-capture")) || - FParse::Param(FCommandLine::Get(), TEXT("message-capture"))) + if (FParse::Param(*CommandLine, TEXT("crash-capture")) || + FParse::Param(*CommandLine, TEXT("message-capture"))) { - RunIntegrationTest(CommandLine); + RunIntegrationTest(*CommandLine); } }