internal: (studio) improve error reporting (#31546)

* internal: (studio) improvements to how studio can access the protocol database

* fix

* fix build

* refactor due to downstream changes

* feat: capture errors with studio

* types

* add tests

* fix types

* fix typescript

* strip path

* fix tests

Author: ryanthemanuel

packages/server/lib/cloud/api/get_and_initialize_studio_manager.ts renamed to packages/server/lib/cloud/api/studio/get_and_initialize_studio_manager.ts

@@ -1,21 +1,21 @@
 import path from 'path'
 import os from 'os'
 import { ensureDir, copy, readFile } from 'fs-extra'
-import { StudioManager } from '../studio'
+import { StudioManager } from '../../studio'
 import tar from 'tar'
-import { verifySignatureFromFile } from '../encryption'
+import { verifySignatureFromFile } from '../../encryption'
 import crypto from 'crypto'
 import fs from 'fs'
 import fetch from 'cross-fetch'
 import { agent } from '@packages/network'
-import { asyncRetry, linearDelay } from '../../util/async_retry'
-import { isRetryableError } from '../network/is_retryable_error'
-import { PUBLIC_KEY_VERSION } from '../constants'
-import { CloudRequest } from './cloud_request'
+import { asyncRetry, linearDelay } from '../../../util/async_retry'
+import { isRetryableError } from '../../network/is_retryable_error'
+import { PUBLIC_KEY_VERSION } from '../../constants'
+import { CloudRequest } from '../cloud_request'
 import type { CloudDataSource } from '@packages/data-context/src/sources'
 
 const pkg = require('@packages/root')
-const routes = require('../routes')
+const routes = require('../../routes')
 
 const _delay = linearDelay(500)
 
@@ -121,17 +121,19 @@ export const retrieveAndExtractStudioBundle = async ({ projectId }: { projectId?
 export const getAndInitializeStudioManager = async ({ projectId, cloudDataSource }: { projectId?: string, cloudDataSource: CloudDataSource }): Promise<StudioManager> => {
   let script: string
 
+  const cloudEnv = (process.env.CYPRESS_INTERNAL_ENV || 'production') as 'development' | 'staging' | 'production'
+  const cloudUrl = cloudDataSource.getCloudUrl(cloudEnv)
+  const cloudHeaders = await cloudDataSource.additionalHeaders()
+
+  let studioHash: string | undefined
+
   try {
-    const { studioHash } = await retrieveAndExtractStudioBundle({ projectId })
+    ({ studioHash } = await retrieveAndExtractStudioBundle({ projectId }))
 
     script = await readFile(serverFilePath, 'utf8')
 
     const studioManager = new StudioManager()
 
-    const cloudEnv = (process.env.CYPRESS_INTERNAL_ENV || 'production') as 'development' | 'staging' | 'production'
-    const cloudUrl = cloudDataSource.getCloudUrl(cloudEnv)
-    const cloudHeaders = await cloudDataSource.additionalHeaders()
-
     await studioManager.setup({
       script,
       studioPath,
@@ -156,7 +158,19 @@ export const getAndInitializeStudioManager = async ({ projectId, cloudDataSource
       actualError = error
     }
 
-    return StudioManager.createInErrorManager(actualError)
+    return StudioManager.createInErrorManager({
+      cloudApi: {
+        cloudUrl,
+        cloudHeaders,
+        CloudRequest,
+        isRetryableError,
+        asyncRetry,
+      },
+      studioHash,
+      projectSlug: projectId,
+      error: actualError,
+      studioMethod: 'getAndInitializeStudioManager',
+    })
   } finally {
     await fs.promises.rm(bundlePath, { force: true })
   }

packages/server/lib/cloud/api/studio/report_studio_error.ts

@@ -0,0 +1,101 @@
+import type { StudioCloudApi } from '@packages/types/src/studio/studio-server-types'
+import Debug from 'debug'
+import { stripPath } from '../../strip_path'
+
+const debug = Debug('cypress:server:cloud:api:studio:report_studio_errors')
+
+export interface ReportStudioErrorOptions {
+  cloudApi: StudioCloudApi
+  studioHash: string | undefined
+  projectSlug: string | undefined
+  error: unknown
+  studioMethod: string
+  studioMethodArgs?: unknown[]
+}
+
+interface StudioError {
+  name: string
+  stack: string
+  message: string
+  studioMethod: string
+  studioMethodArgs?: string
+}
+
+interface StudioErrorPayload {
+  studioHash: string | undefined
+  projectSlug: string | undefined
+  errors: StudioError[]
+}
+
+export function reportStudioError ({
+  cloudApi,
+  studioHash,
+  projectSlug,
+  error,
+  studioMethod,
+  studioMethodArgs,
+}: ReportStudioErrorOptions): void {
+  debug('Error reported:', error)
+
+  // When developing locally, we want to throw the error so we can see it in the console
+  if (process.env.CYPRESS_LOCAL_STUDIO_PATH) {
+    throw error
+  }
+
+  let errorObject: Error
+
+  if (!(error instanceof Error)) {
+    errorObject = new Error(String(error))
+  } else {
+    errorObject = error
+  }
+
+  let studioMethodArgsString: string | undefined
+
+  if (studioMethodArgs) {
+    try {
+      studioMethodArgsString = JSON.stringify({
+        args: studioMethodArgs,
+      })
+    } catch (e: unknown) {
+      studioMethodArgsString = `Unknown args: ${e}`
+    }
+  }
+
+  try {
+    const payload: StudioErrorPayload = {
+      studioHash,
+      projectSlug,
+      errors: [{
+        name: stripPath(errorObject.name ?? `Unknown name`),
+        stack: stripPath(errorObject.stack ?? `Unknown stack`),
+        message: stripPath(errorObject.message ?? `Unknown message`),
+        studioMethod,
+        studioMethodArgs: studioMethodArgsString,
+      }],
+    }
+
+    cloudApi.CloudRequest.post(
+      `${cloudApi.cloudUrl}/studio/errors`,
+      payload,
+      {
+        headers: {
+          'Content-Type': 'application/json',
+          ...cloudApi.cloudHeaders,
+        },
+      },
+    ).catch((e: unknown) => {
+      debug(
+        `Error calling StudioManager.reportError: %o, original error %o`,
+        e,
+        error,
+      )
+    })
+  } catch (e: unknown) {
+    debug(
+      `Error calling StudioManager.reportError: %o, original error %o`,
+      e,
+      error,
+    )
+  }
+}

packages/server/lib/cloud/exception.ts

@@ -4,18 +4,7 @@ const pkg = require('@packages/root')
 const api = require('./api').default
 const user = require('./user')
 const system = require('../util/system')
-
-// strip everything but the file name to remove any sensitive
-// data in the path
-const pathRe = /'?((\/|\\+|[a-z]:\\)[^\s']+)+'?/ig
-const pathSepRe = /[\/\\]+/
-const stripPath = (text) => {
-  return (text || '').replace(pathRe, (path) => {
-    const fileName = _.last(path.split(pathSepRe)) || ''
-
-    return `<stripped-path>${fileName}`
-  })
-}
+const { stripPath } = require('./strip_path')
 
 export = {
   getErr (err: Error) {

packages/server/lib/cloud/strip_path.ts

@@ -0,0 +1,14 @@
+import { last } from 'lodash'
+
+// strip everything but the file name to remove any sensitive
+// data in the path
+const pathRe = /'?((\/|\\+|[a-z]:\\)[^\s']+)+'?/ig
+const pathSepRe = /[\/\\]+/
+
+export const stripPath = (text: string) => {
+  return (text || '').replace(pathRe, (path) => {
+    const fileName = last(path.split(pathSepRe)) || ''
+
+    return `<stripped-path>${fileName}`
+  })
+}

packages/server/lib/cloud/studio.ts

@@ -1,13 +1,10 @@
-import type { StudioErrorReport, StudioManagerShape, StudioStatus, StudioServerDefaultShape, StudioServerShape, ProtocolManagerShape, StudioCloudApi, StudioAIInitializeOptions } from '@packages/types'
+import type { StudioManagerShape, StudioStatus, StudioServerDefaultShape, StudioServerShape, ProtocolManagerShape, StudioCloudApi, StudioAIInitializeOptions } from '@packages/types'
 import type { Router } from 'express'
 import type { Socket } from 'socket.io'
-import fetch from 'cross-fetch'
-import pkg from '@packages/root'
-import os from 'os'
-import { agent } from '@packages/network'
 import Debug from 'debug'
 import { requireScript } from './require_script'
 import path from 'path'
+import { reportStudioError, ReportStudioErrorOptions } from './api/studio/report_studio_error'
 
 interface StudioServer { default: StudioServerDefaultShape }
 
@@ -20,21 +17,26 @@ interface SetupOptions {
 }
 
 const debug = Debug('cypress:server:studio')
-const routes = require('./routes')
 
 export class StudioManager implements StudioManagerShape {
   status: StudioStatus = 'NOT_INITIALIZED'
   isProtocolEnabled: boolean = false
   protocolManager: ProtocolManagerShape | undefined
   private _studioServer: StudioServerShape | undefined
-  private _studioHash: string | undefined
 
-  static createInErrorManager (error: Error): StudioManager {
+  static createInErrorManager ({ cloudApi, studioHash, projectSlug, error, studioMethod, studioMethodArgs }: ReportStudioErrorOptions): StudioManager {
     const manager = new StudioManager()
 
     manager.status = 'IN_ERROR'
 
-    manager.reportError(error).catch(() => { })
+    reportStudioError({
+      cloudApi,
+      studioHash,
+      projectSlug,
+      error,
+      studioMethod,
+      studioMethodArgs,
+    })
 
     return manager
   }
@@ -43,13 +45,13 @@ export class StudioManager implements StudioManagerShape {
     const { createStudioServer } = requireScript<StudioServer>(script).default
 
     this._studioServer = await createStudioServer({
+      studioHash,
       studioPath,
       projectSlug,
       cloudApi,
       betterSqlite3Path: path.dirname(require.resolve('better-sqlite3/package.json')),
     })
 
-    this._studioHash = studioHash
     this.status = 'INITIALIZED'
   }
 
@@ -77,32 +79,11 @@ export class StudioManager implements StudioManagerShape {
     await this.invokeAsync('destroy', { isEssential: true })
   }
 
-  private async reportError (error: Error): Promise<void> {
+  reportError (error: unknown, studioMethod: string, ...studioMethodArgs: unknown[]): void {
     try {
-      const payload: StudioErrorReport = {
-        studioHash: this._studioHash,
-        errors: [{
-          name: error.name ?? `Unknown name`,
-          stack: error.stack ?? `Unknown stack`,
-          message: error.message ?? `Unknown message`,
-        }],
-      }
-
-      const body = JSON.stringify(payload)
-
-      await fetch(routes.apiRoutes.studioErrors() as string, {
-        // @ts-expect-error - this is supported
-        agent,
-        method: 'POST',
-        body,
-        headers: {
-          'Content-Type': 'application/json',
-          'x-cypress-version': pkg.version,
-          'x-os-name': os.platform(),
-          'x-arch': os.arch(),
-        },
-      })
+      this._studioServer?.reportError(error, studioMethod, ...studioMethodArgs)
     } catch (e) {
+      // If we fail to report the error, we shouldn't try and report it again
       debug(`Error calling StudioManager.reportError: %o, original error %o`, e, error)
     }
   }
@@ -129,8 +110,7 @@ export class StudioManager implements StudioManagerShape {
       }
 
       this.status = 'IN_ERROR'
-      // Call and forget this, we don't want to block the main thread
-      this.reportError(actualError).catch(() => { })
+      this.reportError(actualError, method, ...args)
     }
   }
 
@@ -156,10 +136,8 @@ export class StudioManager implements StudioManagerShape {
       }
 
       this.status = 'IN_ERROR'
-      // Call and forget this, we don't want to block the main thread
-      this.reportError(actualError).catch(() => { })
+      this.reportError(actualError, method, ...args)
 
-      // TODO: Figure out errors
       return undefined
     }
   }

packages/server/lib/project-base.ts

@@ -25,7 +25,7 @@ import ProtocolManager from './cloud/protocol'
 import { ServerBase } from './server-base'
 import type Protocol from 'devtools-protocol'
 import type { ServiceWorkerClientEvent } from '@packages/proxy/lib/http/util/service-worker-manager'
-import { getAndInitializeStudioManager } from './cloud/api/get_and_initialize_studio_manager'
+import { getAndInitializeStudioManager } from './cloud/api/studio/get_and_initialize_studio_manager'
 import api from './cloud/api'
 import type { StudioManager } from './cloud/studio'
 import { v4 } from 'uuid'

packages/server/test/support/fixtures/cloud/studio/test-studio.ts

@@ -17,10 +17,14 @@ class StudioServer implements StudioServerShape {
     return Promise.resolve()
   }
 
+  reportError (error: Error, method: string, ...args: any[]): void {
+    // This is a test implementation that does nothing
+  }
+
   destroy (): Promise<void> {
     return Promise.resolve()
   }
-  
+
   addSocketListeners (socket: Socket): void {
     // This is a test implementation that does nothing
   }