[js/web] Prepare to integrate ONNX Runtime Web CI with BrowserStack (#…

…8843)

* Integrate BrowserStack with ONNX Runtime Web CI pipeline

* Change to Linux command for BrowserStack CI

* Set preferTriggeringPipeline as true

* Fix a commit fetching script

* Remove wasm binary download from the latest build

* Use release build of WebAssembly

* Disable check-out of commit for testing

* Use commit of WebAssembly build CI pipeline

* Need to issue two PRs to prevent build failure

js/web/karma.conf.js

@@ -39,10 +39,11 @@ module.exports = function (config) {
   config.set({
     // global config of your BrowserStack account
     browserStack: {
-      username: process.env.BROWSER_STACK_USERNAME,
-      accessKey: process.env.BROWSER_STACK_ACCESS_KEY,
+      username: process.env.BROWSERSTACK_USERNAME,
+      accessKey: process.env.BROWSERSTACK_ACCESS_KEY,
       forceLocal: true,
       startTunnel: true,
+      idleTimeout: '300',
     },
     frameworks: ['mocha'],
     files: [
@@ -78,105 +79,92 @@ module.exports = function (config) {
     customLaunchers: {
       ChromeTest: { base: 'Chrome', flags: ['--window-size=1,1', '--enable-features=SharedArrayBuffer'] },
       ChromeDebug: { debug: true, base: 'Chrome', flags: ['--remote-debugging-port=9333', '--enable-features=SharedArrayBuffer'] },
+
       //
       // ==== BrowserStack browsers ====
       //
 
       // Windows
       //
-      BS_WIN_10_Chrome_73: {
+      BS_WIN_10_Chrome_91: {
         base: 'BrowserStack',
-        browser: 'Chrome',
-        browser_version: '73.0',
         os: 'Windows',
         os_version: '10',
+        browser: 'Chrome',
+        browser_version: '91'
       },
-      BS_WIN_10_Edge_18: {
+      BS_WIN_10_Edge_91: {
         base: 'BrowserStack',
         os: 'Windows',
         os_version: '10',
         browser: 'Edge',
-        browser_version: '18.0',
+        browser_version: '91'
       },
-      BS_WIN_10_Firefox_66: {
+      BS_WIN_10_Firefox_89: {
         base: 'BrowserStack',
         os: 'Windows',
         os_version: '10',
         browser: 'Firefox',
-        browser_version: '66.0',
-      },
-      BS_WIN_7_Chrome_63: {
-        base: 'BrowserStack',
-        browser: 'Chrome',
-        browser_version: '63.0',
-        os: 'Windows',
-        os_version: '7',
+        browser_version: '89'
       },
 
       // macOS
       //
-      BS_MAC_10_14_Safari_12: {
+      BS_MAC_11_Safari_14: {
         base: 'BrowserStack',
         os: 'OS X',
-        os_version: 'Mojave',
+        os_version: 'Big Sur',
         browser: 'Safari',
-        browser_version: '12.0',
+        browser_version: '14.0'
       },
-      BS_MAC_10_14_Chrome_73: {
+      BS_MAC_11_Chrome_91: {
         base: 'BrowserStack',
         os: 'OS X',
-        os_version: 'Mojave',
+        os_version: 'Big Sur',
         browser: 'Chrome',
-        browser_version: '73.0',
-      },
-      BS_MAC_10_13_Safari_11_1: {
-        base: 'BrowserStack',
-        os: 'OS X',
-        os_version: 'High Sierra',
-        browser: 'Safari',
-        browser_version: '11.1',
+        browser_version: '91'
       },
 
       // iPhone
       //
-      BS_IOS_12_1_iPhoneXS: {
+      BS_IOS_14_iPhoneXS: {
         base: 'BrowserStack',
         device: 'iPhone XS',
         real_mobile: true,
         os: 'ios',
-        os_version: '12.1',
-      },
-      BS_IOS_11_iPhoneX: {
-        base: 'BrowserStack',
-        device: 'iPhone X',
-        real_mobile: true,
-        os: 'ios',
-        os_version: '11',
+        os_version: '14'
       },
-      BS_IOS_10_3_iPhone7: {
+      BS_IOS_13_iPhoneXS: {
         base: 'BrowserStack',
-        device: 'iPhone 7',
+        device: 'iPhone XS',
         real_mobile: true,
         os: 'ios',
-        os_version: '10.3',
+        os_version: '13'
       },
 
       // Android
       //
-      BS_ANDROID_9_Pixel_3: {
+      BS_ANDROID_11_Pixel_5: {
         base: 'BrowserStack',
-        device: 'Google Pixel 3',
+        device: 'Google Pixel 5',
         real_mobile: true,
         os: 'android',
-        os_version: '9.0',
+        os_version: '11.0'
       },
-      BS_ANDROID_7_1_Galaxy_Note_8: {
+      BS_ANDROID_11_Galaxy_S_21: {
         base: 'BrowserStack',
-        device: 'Samsung Galaxy Note 8',
+        device: 'Samsung Galaxy S21',
         real_mobile: true,
         os: 'android',
-        os_version: '7.1',
+        os_version: '11.0'
       },
+      BS_ANDROID_10_Pixel_4: {
+        base: 'BrowserStack',
+        device: 'Google Pixel 4',
+        real_mobile: true,
+        os: 'android',
+        os_version: '10.0'
+      }
     }
   });
 };

js/web/script/test-runner-cli-args.ts

@@ -19,6 +19,7 @@ Usage:
 
 Modes:
  suite0                        Run all unittests, all operator tests and node model tests that described in suite test list
+ suite1                        Run all operator tests and node model tests that described in suite test list
  model                         Run a single model test
  unittest                      Run all unittests
  op                            Run a single operator test
@@ -94,7 +95,7 @@ Examples:
 /* eslint-enable max-len */
 
 export declare namespace TestRunnerCliArgs {
-  type Mode = 'suite0'|'model'|'unittest'|'op';
+  type Mode = 'suite0'|'suite1'|'model'|'unittest'|'op';
   type Backend = 'cpu'|'webgl'|'wasm'|'onnxruntime';
   type Environment = 'chrome'|'edge'|'firefox'|'electron'|'safari'|'node'|'bs';
   type BundleMode = 'prod'|'dev'|'perf';

js/web/script/test-runner-cli.ts

@@ -33,7 +33,7 @@ const TEST_DATA_OP_ROOT = path.join(TEST_ROOT, 'data', 'ops');
 const TEST_DATA_BASE = args.env === 'node' ? TEST_ROOT : '/base/test/';
 
 let testlist: Test.TestList;
-const shouldLoadSuiteTestData = (args.mode === 'suite0');
+const shouldLoadSuiteTestData = (args.mode === 'suite0' || args.mode === 'suite1');
 if (shouldLoadSuiteTestData) {
   npmlog.verbose('TestRunnerCli.Init', 'Loading testlist...');
 
@@ -90,13 +90,16 @@ let unittest = false;
 npmlog.verbose('TestRunnerCli.Init', 'Preparing test config...');
 switch (args.mode) {
   case 'suite0':
+  case 'suite1':
     for (const backend of DEFAULT_BACKENDS) {
       if (args.backends.indexOf(backend) !== -1) {
         modelTestGroups.push(...nodeTests.get(backend)!);  // model test : node
         opTestGroups.push(...opTests.get(backend)!);       // operator test
       }
     }
-    unittest = true;
+    if (args.mode === 'suite0') {
+      unittest = true;
+    }
     break;
 
   case 'model':