Build/Test Tools: Publish PHPUnit timing metrics to CodeVitals

.github/workflows/phpunit-tests.yml

@@ -81,6 +81,7 @@ jobs:
     permissions:
       contents: read
     secrets:
+      CODEVITALS_PROJECT_TOKEN: ${{ secrets.CODEVITALS_PROJECT_TOKEN }}
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
       WPT_REPORT_API_KEY: ${{ secrets.WPT_REPORT_API_KEY }}
     if: |

.github/workflows/reusable-phpunit-tests-v3.yml

@@ -83,6 +83,9 @@ on:
         type: string
         default: ''
     secrets:
+      CODEVITALS_PROJECT_TOKEN:
+        description: 'The authorization token for publishing results to CodeVitals.'
+        required: false
       CODECOV_TOKEN:
         description: 'The Codecov token required for uploading reports.'
         required: false
@@ -123,6 +126,7 @@ jobs:
   # - Logs debug information about what's installed within the WordPress Docker containers.
   # - Install WordPress within the Docker container.
   # - Run the PHPUnit tests.
+  # - Publish PHPUnit timing metrics to CodeVitals.
   # - Upload the code coverage report to Codecov.io.
   # - Ensures version-controlled files are not modified or deleted.
   # - Checks out the WordPress Test reporter repository.
@@ -269,6 +273,40 @@ jobs:
           TEST_GROUPS: ${{ inputs.phpunit-test-groups }}
           MULTISITE_FLAG: ${{ inputs.multisite && 'multisite' || 'single' }}
 
+      - name: Publish PHPUnit timing metrics
+        continue-on-error: true
+        if: |
+          github.event_name == 'push' &&
+          github.ref == 'refs/heads/trunk' &&
+          inputs.php == '8.5' &&
+          inputs.report
+        env:
+          CODEVITALS_PROJECT_TOKEN: ${{ secrets.CODEVITALS_PROJECT_TOKEN }}
+        shell: bash
+        run: |
+          if [ -z "$CODEVITALS_PROJECT_TOKEN" ]; then
+            echo "PHPUnit timing metrics could not be published. 'CODEVITALS_PROJECT_TOKEN' is not set"
+            exit 1
+          fi
+          COMMITTED_AT="$(git show -s "$GITHUB_SHA" --format='%cI')"
+          RESPONSE="$(
+            php tests/phpunit/prepare-timing-results.php \
+              tests/phpunit/build/logs/junit.xml \
+              trunk \
+              "$GITHUB_SHA" \
+              "$COMMITTED_AT" \
+              | curl --fail-with-body --silent --show-error \
+                --request POST \
+                --header 'Content-Type: application/json' \
+                --data-binary @- \
+                "https://codevitals.run/api/log?token=${CODEVITALS_PROJECT_TOKEN}"
+          )"
+          if ! jq --exit-status '.status == "ok" and .count == 6' <<< "$RESPONSE" > /dev/null; then
+            echo 'CodeVitals did not accept all six PHPUnit timing metrics.'
+            exit 1
+          fi
+          echo 'Published six PHPUnit timing metrics to CodeVitals.'
+
       - name: Run AJAX tests
         if: ${{ ! inputs.phpunit-test-groups && ! inputs.coverage-report }}
         continue-on-error: ${{ inputs.allow-errors }}

tests/phpunit/includes/class-wp-phpunit-timing-metrics.php

@@ -0,0 +1,88 @@
+<?php
+
+/**
+ * Extracts aggregate timing metrics from a PHPUnit JUnit report.
+ */
+final class WP_PHPUnit_Timing_Metrics {
+
+	/**
+	 * Extracts timing metrics from a JUnit XML file.
+	 *
+	 * @param string $file Path to the JUnit XML file.
+	 * @return array<string, float|int> Timing metrics keyed for CodeVitals.
+	 * @throws RuntimeException If the file cannot be read or contains invalid timing data.
+	 */
+	public static function from_file( $file ) {
+		if ( ! is_readable( $file ) ) {
+			throw new RuntimeException( 'The JUnit timing report could not be read.' );
+		}
+
+		$reader = new XMLReader();
+		if ( ! $reader->open( $file, null, LIBXML_NONET | LIBXML_COMPACT ) ) {
+			throw new RuntimeException( 'The JUnit timing report could not be opened.' );
+		}
+
+		$suite_time = null;
+		$test_times = array();
+
+		while ( $reader->read() ) {
+			if ( XMLReader::ELEMENT !== $reader->nodeType ) {
+				continue;
+			}
+
+			if ( null === $suite_time && 'testsuite' === $reader->name ) {
+				$time = $reader->getAttribute( 'time' );
+				if ( is_numeric( $time ) ) {
+					$suite_time = (float) $time;
+				}
+				continue;
+			}
+
+			if ( 'testcase' !== $reader->name ) {
+				continue;
+			}
+
+			$time = $reader->getAttribute( 'time' );
+			if ( ! is_numeric( $time ) ) {
+				$reader->close();
+				throw new RuntimeException( 'A JUnit testcase is missing numeric timing data.' );
+			}
+
+			$test_times[] = (float) $time;
+		}
+
+		$reader->close();
+
+		if ( ! $test_times ) {
+			throw new RuntimeException( 'The JUnit timing report contains no testcases.' );
+		}
+
+		if ( null === $suite_time ) {
+			$suite_time = array_sum( $test_times );
+		}
+
+		sort( $test_times, SORT_NUMERIC );
+
+		return array(
+			'phpunit-suite-time'       => round( $suite_time, 6 ),
+			'phpunit-p95-test-time'    => round( self::percentile( $test_times, 0.95 ) * 1000, 3 ),
+			'phpunit-p99-test-time'    => round( self::percentile( $test_times, 0.99 ) * 1000, 3 ),
+			'phpunit-max-test-time'    => round( max( $test_times ) * 1000, 3 ),
+			'phpunit-tests-over-500ms' => count( array_filter( $test_times, static fn ( $time ) => $time > 0.5 ) ),
+			'phpunit-tests-over-1s'    => count( array_filter( $test_times, static fn ( $time ) => $time > 1 ) ),
+		);
+	}
+
+	/**
+	 * Calculates a nearest-rank percentile from a sorted list.
+	 *
+	 * @param float[] $values     Sorted values.
+	 * @param float   $percentile Percentile between zero and one.
+	 * @return float Percentile value.
+	 */
+	private static function percentile( $values, $percentile ) {
+		$index = (int) ceil( $percentile * count( $values ) ) - 1;
+
+		return $values[ max( 0, $index ) ];
+	}
+}

tests/phpunit/prepare-timing-results.php

@@ -0,0 +1,33 @@
+#!/usr/bin/env php
+<?php
+
+/**
+ * Prepares aggregate PHPUnit timing metrics for publication to CodeVitals.
+ *
+ * @package WordPress
+ * @subpackage UnitTests
+ */
+
+require_once __DIR__ . '/includes/class-wp-phpunit-timing-metrics.php';
+
+if ( 5 !== $argc ) {
+	fwrite( STDERR, "Usage: prepare-timing-results.php <junit-file> <branch> <hash> <timestamp>\n" );
+	exit( 1 );
+}
+
+try {
+	$timestamp = new DateTimeImmutable( $argv[4] );
+	$payload   = array(
+		'branch'      => $argv[2],
+		'hash'        => $argv[3],
+		'baseHash'    => $argv[3],
+		'baseMetrics' => new stdClass(),
+		'timestamp'   => $timestamp->format( DATE_ATOM ),
+		'metrics'     => WP_PHPUnit_Timing_Metrics::from_file( $argv[1] ),
+	);
+
+	echo json_encode( $payload, JSON_THROW_ON_ERROR ) . "\n";
+} catch ( Throwable $error ) {
+	fwrite( STDERR, $error->getMessage() . "\n" );
+	exit( 1 );
+}

tests/phpunit/tests/includes/junitTimingMetrics.php

@@ -0,0 +1,102 @@
+<?php
+
+require_once dirname( __DIR__, 2 ) . '/includes/class-wp-phpunit-timing-metrics.php';
+
+/**
+ * @group core-vitals-tooling
+ *
+ * @ticket 65887
+ */
+class Tests_Includes_JUnit_Timing_Metrics extends WP_UnitTestCase {
+
+	/**
+	 * Temporary files created by a test.
+	 *
+	 * @var string[]
+	 */
+	private $temporary_files = array();
+
+	public function tear_down() {
+		foreach ( $this->temporary_files as $file ) {
+			unlink( $file );
+		}
+
+		parent::tear_down();
+	}
+
+	public function test_extracts_aggregate_timing_metrics() {
+		$times = array_map(
+			static fn ( $millisecond ) => $millisecond / 1000,
+			range( 1, 100 )
+		);
+		$file  = $this->create_junit_file( $times, 5.05 );
+
+		$this->assertSame(
+			array(
+				'phpunit-suite-time'       => 5.05,
+				'phpunit-p95-test-time'    => 95.0,
+				'phpunit-p99-test-time'    => 99.0,
+				'phpunit-max-test-time'    => 100.0,
+				'phpunit-tests-over-500ms' => 0,
+				'phpunit-tests-over-1s'    => 0,
+			),
+			WP_PHPUnit_Timing_Metrics::from_file( $file )
+		);
+	}
+
+	public function test_counts_only_tests_above_slow_test_thresholds() {
+		$file = $this->create_junit_file( array( 0.5, 0.500001, 1.0, 1.000001 ), 3.000002 );
+
+		$metrics = WP_PHPUnit_Timing_Metrics::from_file( $file );
+
+		$this->assertSame( 3, $metrics['phpunit-tests-over-500ms'] );
+		$this->assertSame( 1, $metrics['phpunit-tests-over-1s'] );
+	}
+
+	public function test_uses_testcase_time_when_suite_time_is_missing() {
+		$file = $this->create_junit_file( array( 0.1, 0.2, 0.3 ) );
+
+		$metrics = WP_PHPUnit_Timing_Metrics::from_file( $file );
+
+		$this->assertSame( 0.6, $metrics['phpunit-suite-time'] );
+	}
+
+	public function test_rejects_report_without_testcases() {
+		$file = $this->create_junit_file( array(), 0.0 );
+
+		$this->expectException( RuntimeException::class );
+		$this->expectExceptionMessage( 'The JUnit timing report contains no testcases.' );
+
+		WP_PHPUnit_Timing_Metrics::from_file( $file );
+	}
+
+	/**
+	 * Creates a JUnit XML file for a test.
+	 *
+	 * @param float[]   $times      Testcase times in seconds.
+	 * @param float|int $suite_time Optional testsuite time in seconds.
+	 * @return string Path to the temporary file.
+	 */
+	private function create_junit_file( $times, $suite_time = null ) {
+		$file = tempnam( sys_get_temp_dir(), 'junit-timing-' );
+
+		if ( false === $file ) {
+			$this->fail( 'Failed to create a temporary JUnit file.' );
+		}
+
+		$this->temporary_files[] = $file;
+		$suite_time_attribute    = null === $suite_time ? '' : sprintf( ' time="%s"', $suite_time );
+		$testcases               = '';
+
+		foreach ( $times as $index => $time ) {
+			$testcases .= sprintf( '<testcase name="test_%1$d" time="%2$s"/>', $index, $time );
+		}
+
+		file_put_contents(
+			$file,
+			sprintf( '<?xml version="1.0"?><testsuites><testsuite tests="%1$d"%2$s>%3$s</testsuite></testsuites>', count( $times ), $suite_time_attribute, $testcases )
+		);
+
+		return $file;
+	}
+}