diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 21cfac5ade..636b9bdd32 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -34,7 +34,7 @@ jobs: - 'src/wasm-lib/**' playwright-chrome: - timeout-minutes: ${{ matrix.os == 'macos-14' && 60 || 40 }} + timeout-minutes: ${{ matrix.os == 'macos-14' && 60 || 50 }} strategy: fail-fast: false matrix: @@ -232,6 +232,7 @@ jobs: exit 0 env: CI: true + FAIL_ON_CONSOLE_ERRORS: true NODE_ENV: development VITE_KC_DEV_TOKEN: ${{ secrets.KITTYCAD_API_TOKEN_DEV }} VITE_KC_SKIP_AUTH: true @@ -410,6 +411,7 @@ jobs: exit 0 env: CI: true + FAIL_ON_CONSOLE_ERRORS: true NODE_ENV: development VITE_KC_DEV_TOKEN: ${{ secrets.KITTYCAD_API_TOKEN_DEV }} VITE_KC_SKIP_AUTH: true diff --git a/e2e/playwright/basic-sketch.spec.ts b/e2e/playwright/basic-sketch.spec.ts index 1463bee7f0..37920eb3f2 100644 --- a/e2e/playwright/basic-sketch.spec.ts +++ b/e2e/playwright/basic-sketch.spec.ts @@ -8,8 +8,8 @@ import { PERSIST_MODELING_CONTEXT, } from './test-utils' -test.beforeEach(async ({ context, page }) => { - await setup(context, page) +test.beforeEach(async ({ context, page }, testInfo) => { + await setup(context, page, testInfo) }) test.afterEach(async ({ page }, testInfo) => { diff --git a/e2e/playwright/can-create-sketches-on-all-planes-and-their-back-sides.spec.ts b/e2e/playwright/can-create-sketches-on-all-planes-and-their-back-sides.spec.ts index 272504fea0..f64294b6f9 100644 --- a/e2e/playwright/can-create-sketches-on-all-planes-and-their-back-sides.spec.ts +++ b/e2e/playwright/can-create-sketches-on-all-planes-and-their-back-sides.spec.ts @@ -3,8 +3,8 @@ import { getUtils, setup, tearDown } from './test-utils' import { EngineCommand } from 'lang/std/artifactGraph' import { uuidv4 } from 'lib/utils' -test.beforeEach(async ({ context, page }) => { - await setup(context, page) +test.beforeEach(async ({ context, page }, testInfo) => { + await setup(context, page, testInfo) }) test.afterEach(async ({ page }, testInfo) => { diff --git a/e2e/playwright/code-pane-and-errors.spec.ts b/e2e/playwright/code-pane-and-errors.spec.ts index b6e1dd8ad8..67605076ea 100644 --- a/e2e/playwright/code-pane-and-errors.spec.ts +++ b/e2e/playwright/code-pane-and-errors.spec.ts @@ -12,8 +12,8 @@ import { bracket } from 'lib/exampleKcl' import { TEST_CODE_LONG_WITH_ERROR_OUT_OF_VIEW } from './storageStates' import fsp from 'fs/promises' -test.beforeEach(async ({ context, page }) => { - await setup(context, page) +test.beforeEach(async ({ context, page }, testInfo) => { + await setup(context, page, testInfo) }) test.afterEach(async ({ page }, testInfo) => { diff --git a/e2e/playwright/command-bar-tests.spec.ts b/e2e/playwright/command-bar-tests.spec.ts index 3d4a6ca7ea..5be7320bf1 100644 --- a/e2e/playwright/command-bar-tests.spec.ts +++ b/e2e/playwright/command-bar-tests.spec.ts @@ -3,8 +3,8 @@ import { test, expect } from '@playwright/test' import { getUtils, setup, tearDown } from './test-utils' import { KCL_DEFAULT_LENGTH } from 'lib/constants' -test.beforeEach(async ({ context, page }) => { - await setup(context, page) +test.beforeEach(async ({ context, page }, testInfo) => { + await setup(context, page, testInfo) }) test.afterEach(async ({ page }, testInfo) => { diff --git a/e2e/playwright/copilot-ghost-test.spec.ts b/e2e/playwright/copilot-ghost-test.spec.ts index 2ba2ac4079..027ceeabcb 100644 --- a/e2e/playwright/copilot-ghost-test.spec.ts +++ b/e2e/playwright/copilot-ghost-test.spec.ts @@ -1,8 +1,8 @@ import { test, expect } from '@playwright/test' import { getUtils, setup, tearDown } from './test-utils' -test.beforeEach(async ({ context, page }) => { - await setup(context, page) +test.beforeEach(async ({ context, page }, testInfo) => { + await setup(context, page, testInfo) }) test.afterEach(async ({ page }, testInfo) => { diff --git a/e2e/playwright/editor-tests.spec.ts b/e2e/playwright/editor-tests.spec.ts index 82c5e84886..d065076774 100644 --- a/e2e/playwright/editor-tests.spec.ts +++ b/e2e/playwright/editor-tests.spec.ts @@ -2,8 +2,8 @@ import { test, expect } from '@playwright/test' import { uuidv4 } from 'lib/utils' import { getUtils, setup, tearDown } from './test-utils' -test.beforeEach(async ({ context, page }) => { - await setup(context, page) +test.beforeEach(async ({ context, page }, testInfo) => { + await setup(context, page, testInfo) }) test.afterEach(async ({ page }, testInfo) => { diff --git a/e2e/playwright/file-tree.spec.ts b/e2e/playwright/file-tree.spec.ts index a33013f161..f72a2a6663 100644 --- a/e2e/playwright/file-tree.spec.ts +++ b/e2e/playwright/file-tree.spec.ts @@ -2,8 +2,8 @@ import { test, expect } from '@playwright/test' import * as fsp from 'fs/promises' import { getUtils, setup, setupElectron, tearDown } from './test-utils' -test.beforeEach(async ({ context, page }) => { - await setup(context, page) +test.beforeEach(async ({ context, page }, testInfo) => { + await setup(context, page, testInfo) }) test.afterEach(async ({ page }, testInfo) => { @@ -108,7 +108,6 @@ test.describe('when using the file tree to', () => { async ({ browser: _ }, testInfo) => { const { electronApp, page } = await setupElectron({ testInfo, - folderSetupFn: async () => {}, }) const { @@ -151,6 +150,7 @@ test.describe('when using the file tree to', () => { await selectFile(kcl1) await editorTextMatches(kclCube) }) + await page.waitForTimeout(500) await test.step(`Postcondition: ${kcl2} still exists with the original content`, async () => { await selectFile(kcl2) diff --git a/e2e/playwright/lib/console-error-whitelist.ts b/e2e/playwright/lib/console-error-whitelist.ts new file mode 100644 index 0000000000..653dff0be8 --- /dev/null +++ b/e2e/playwright/lib/console-error-whitelist.ts @@ -0,0 +1,270 @@ +export const isErrorWhitelisted = (exception: Error) => { + // due to the way webkit/Google Chrome report errors, it was necessary + // to whitelist similar errors separately for each project + let whitelist: { + name: string + message: string + stack: string + foundInSpec: string + project: 'webkit' | 'Google Chrome' + }[] = [ + { + name: '', + message: 'undefined', + stack: '', + foundInSpec: `e2e/playwright/sketch-tests.spec.ts Existing sketch with bad code delete user's code`, + project: 'Google Chrome', + }, + { + name: '"{"kind"', + message: + '"engine","sourceRanges":[[0,0]],"msg":"Failed to get string from response from engine: `JsValue(undefined)`"}"', + stack: '', + foundInSpec: 'e2e/playwright/testing-settings.spec.ts', + project: 'Google Chrome', + }, + { + name: '', + message: 'false', + stack: '', + foundInSpec: 'e2e/playwright/testing-segment-overlays.spec.ts', + project: 'Google Chrome', + }, + { + name: '{"kind"', + // eslint-disable-next-line no-useless-escape + message: 'no connection to send on', + stack: '', + foundInSpec: 'e2e/playwright/various.spec.ts', + project: 'Google Chrome', + }, + { + name: '', + message: 'sketchGroup not found', + stack: '', + foundInSpec: + 'e2e/playwright/testing-selections.spec.ts Deselecting line tool should mean nothing happens on click', + project: 'Google Chrome', + }, + { + name: 'engine error', + message: + '[{"error_code":"bad_request","message":"Cannot set the camera position with these values"}]', + stack: '', + foundInSpec: + 'e2e/playwright/can-create-sketches-on-all-planes-and-their-back-sides.spec.ts XY', + project: 'Google Chrome', + }, + { + name: '', + message: 'no connection to send on', + stack: '', + foundInSpec: + 'e2e/playwright/can-create-sketches-on-all-planes-and-their-back-sides.spec.ts XY', + project: 'Google Chrome', + }, + { + name: 'RangeError', + message: 'Position 160 is out of range for changeset of length 0', + stack: `RangeError: Position 160 is out of range for changeset of length 0 + at _ChangeSet.mapPos (http://localhost:3000/node_modules/.vite/deps/chunk-3BHLKIA4.js?v=412eae63:756:13) + at findSharedChunks (http://localhost:3000/node_modules/.vite/deps/chunk-3BHLKIA4.js?v=412eae63:3045:49) + at _RangeSet.compare (http://localhost:3000/node_modules/.vite/deps/chunk-3BHLKIA4.js?v=412eae63:2840:24) + at findChangedDeco (http://localhost:3000/node_modules/.vite/deps/chunk-IZYF444B.js?v=412eae63:3320:12) + at DocView.update (http://localhost:3000/node_modules/.vite/deps/chunk-IZYF444B.js?v=412eae63:2774:20) + at _EditorView.update (http://localhost:3000/node_modules/.vite/deps/chunk-IZYF444B.js?v=412eae63:7056:30) + at DOMObserver.flush (http://localhost:3000/node_modules/.vite/deps/chunk-IZYF444B.js?v=412eae63:6621:17) + at MutationObserver. (http://localhost:3000/node_modules/.vite/deps/chunk-IZYF444B.js?v=412eae63:6322:14)`, + foundInSpec: 'e2e/playwright/editor-tests.spec.ts fold gutters work', + project: 'Google Chrome', + }, + { + name: 'RangeError', + message: 'Selection points outside of document', + stack: `RangeError: Selection points outside of document + + at checkSelection (http://localhost:3000/node_modules/.vite/deps/chunk-3BHLKIA4.js?v=412eae63:1453:13) + + at new _Transaction (http://localhost:3000/node_modules/.vite/deps/chunk-3BHLKIA4.js?v=412eae63:2014:7) + + at _Transaction.create (http://localhost:3000/node_modules/.vite/deps/chunk-3BHLKIA4.js?v=412eae63:2022:12) + + at resolveTransaction (http://localhost:3000/node_modules/.vite/deps/chunk-3BHLKIA4.js?v=412eae63:2155:24) + + at _EditorState.update (http://localhost:3000/node_modules/.vite/deps/chunk-3BHLKIA4.js?v=412eae63:2281:12) + + at _EditorView.dispatch (http://localhost:3000/node_modules/.vite/deps/chunk-IZYF444B.js?v=412eae63:6988:148) + + at EditorManager.selectRange (http://localhost:3000/src/editor/manager.ts:182:22) + + at AST extrude (http://localhost:3000/src/machines/modelingMachine.ts:828:25)`, + foundInSpec: 'e2e/playwright/editor-tests.spec.ts', + project: 'Google Chrome', + }, + { + name: 'Unhandled Promise Rejection', + message: "TypeError: null is not an object (evaluating 'sg.value')", + stack: `Unhandled Promise Rejection: TypeError: null is not an object (evaluating 'sg.value') + at unknown (http://localhost:3000/src/clientSideScene/sceneEntities.ts:466:23) + at unknown (http://localhost:3000/src/clientSideScene/sceneEntities.ts:454:32) + at set up draft line without teardown (http://localhost:3000/src/machines/modelingMachine.ts:983:47) + at unknown (http://localhost:3000/node_modules/.vite/deps/chunk-6FRHHHSJ.js?v=0de2e74f:1877:24) + at handleAction (http://localhost:3000/node_modules/.vite/deps/chunk-6FRHHHSJ.js?v=0de2e74f:1064:26) + at processBlock (http://localhost:3000/node_modules/.vite/deps/chunk-6FRHHHSJ.js?v=0de2e74f:1087:36) + at map ([native code]:0:0) + at resolveActions (http://localhost:3000/node_modules/.vite/deps/chunk-6FRHHHSJ.js?v=0de2e74f:1109:49) + at unknown (http://localhost:3000/node_modules/.vite/deps/chunk-6FRHHHSJ.js?v=0de2e74f:3639:37) + at provide (http://localhost:3000/node_modules/.vite/deps/chunk-6FRHHHSJ.js?v=0de2e74f:1117:18) + at unknown (http://localhost:3000/node_modules/.vite/deps/chunk-6FRHHHSJ.js?v=0de2e74f:2452:30) + at unknown (http://localhost:3000/node_modules/.vite/deps/chunk-6FRHHHSJ.js?v=0de2e74f:1831:43) + at unknown (http://localhost:3000/node_modules/.vite/deps/chunk-6FRHHHSJ.js?v=0de2e74f:1659:17) + at unknown (http://localhost:3000/node_modules/.vite/deps/chunk-6FRHHHSJ.js?v=0de2e74f:1643:19) + at unknown (http://localhost:3000/node_modules/.vite/deps/chunk-6FRHHHSJ.js?v=0de2e74f:1829:33) + at unknown (http://localhost:3000/src/clientSideScene/sceneEntities.ts:263:19)`, + foundInSpec: `e2e/playwright/testing-camera-movement.spec.ts Zoom should be consistent when exiting or entering sketches`, + project: 'webkit', + }, + { + name: 'Unhandled Promise Rejection', + message: 'false', + stack: `Unhandled Promise Rejection: false + at unknown (http://localhost:3000/src/clientSideScene/ClientSideSceneComp.tsx:455:78)`, + foundInSpec: `e2e/playwright/testing-segment-overlays.spec.ts line-[tagOutsideSketch]`, + project: 'webkit', + }, + { + name: 'Unhandled Promise Rejection', + message: `TypeError: null is not an object (evaluating 'programMemory.get(variableDeclarationName).value')`, + stack: ` + stack:Unhandled Promise Rejection: TypeError: null is not an object (evaluating 'programMemory.get(variableDeclarationName).value') + + at unknown (http://localhost:3000/src/machines/modelingMachine.ts:911:49)`, + foundInSpec: `e2e/playwright/can-create-sketches-on-all-planes-and-their-back-sides.spec.ts`, + project: 'webkit', + }, + { + name: 'Unhandled Promise Rejection', + message: `null is not an object (evaluating 'programMemory.get(variableDeclarationName).value')`, + stack: `Unhandled Promise Rejection: TypeError: null is not an object (evaluating 'programMemory.get(variableDeclarationName).value') + at unknown (http://localhost:3000/src/machines/modelingMachine.ts:911:49)`, + foundInSpec: `e2e/playwright/testing-camera-movement.spec.ts Zoom should be consistent when exiting or entering sketches`, + project: 'webkit', + }, + { + name: 'TypeError', + message: `null is not an object (evaluating 'gl.getShaderPrecisionFormat(gl.VERTEX_SHADER, gl.HIGH_FLOAT).precision')`, + stack: `TypeError: null is not an object (evaluating 'gl.getShaderPrecisionFormat(gl.VERTEX_SHADER, gl.HIGH_FLOAT).precision') + at getMaxPrecision (http://localhost:3000/node_modules/.vite/deps/chunk-DEEFU7IG.js?v=d328572b:9557:71) + at WebGLCapabilities (http://localhost:3000/node_modules/.vite/deps/chunk-DEEFU7IG.js?v=d328572b:9570:39) + at initGLContext (http://localhost:3000/node_modules/.vite/deps/chunk-DEEFU7IG.js?v=d328572b:16993:43) + at WebGLRenderer (http://localhost:3000/node_modules/.vite/deps/chunk-DEEFU7IG.js?v=d328572b:17024:18) + at SceneInfra (http://localhost:3000/src/clientSideScene/sceneInfra.ts:185:38) + at module code (http://localhost:3000/src/lib/singletons.ts:14:41)`, + foundInSpec: `e2e/playwright/testing-segment-overlays.spec.ts angledLineToX`, + project: 'webkit', + }, + { + name: 'Unhandled Promise Rejection', + message: + '{"kind":"engine","sourceRanges":[[0,0]],"msg":"Failed to get string from response from engine: `JsValue(undefined)`"}', + stack: `Unhandled Promise Rejection: {"kind":"engine","sourceRanges":[[0,0]],"msg":"Failed to get string from response from engine: \`JsValue(undefined)\`"} + at unknown (http://localhost:3000/src/lang/std/engineConnection.ts:1245:26)`, + foundInSpec: + 'e2e/playwright/onboarding-tests.spec.ts Click through each onboarding step', + project: 'webkit', + }, + { + name: 'Unhandled Promise Rejection', + message: 'undefined', + stack: '', + foundInSpec: `e2e/playwright/sketch-tests.spec.ts Existing sketch with bad code delete user's code`, + project: 'webkit', + }, + { + name: 'Fetch API cannot load https', + message: '/api.dev.zoo.dev/logout due to access control checks.', + stack: `Fetch API cannot load https://api.dev.zoo.dev/logout due to access control checks. + at goToSignInPage (http://localhost:3000/src/components/SettingsAuthProvider.tsx:229:15) + at unknown (http://localhost:3000/node_modules/.vite/deps/chunk-6FRHHHSJ.js?v=d328572b:1877:24) + at handleAction (http://localhost:3000/node_modules/.vite/deps/chunk-6FRHHHSJ.js?v=d328572b:1064:26) + at processBlock (http://localhost:3000/node_modules/.vite/deps/chunk-6FRHHHSJ.js?v=d328572b:1087:36) + at map (:1:11) + at resolveActions (http://localhost:3000/node_modules/.vite/deps/chunk-6FRHHHSJ.js?v=d328572b:1109:49) + at unknown (http://localhost:3000/node_modules/.vite/deps/chunk-6FRHHHSJ.js?v=d328572b:3639:37) + at provide (http://localhost:3000/node_modules/.vite/deps/chunk-6FRHHHSJ.js?v=d328572b:1117:18) + at unknown (http://localhost:3000/node_modules/.vite/deps/chunk-6FRHHHSJ.js?v=d328572b:2452:30) + at unknown (http://localhost:3000/node_modules/.vite/deps/chunk-6FRHHHSJ.js?v=d328572b:1831:43) + at unknown (http://localhost:3000/node_modules/.vite/deps/chunk-6FRHHHSJ.js?v=d328572b:1659:17) + at unknown (http://localhost:3000/node_modules/.vite/deps/chunk-6FRHHHSJ.js?v=d328572b:1643:19) + at unknown (http://localhost:3000/node_modules/.vite/deps/chunk-6FRHHHSJ.js?v=d328572b:1829:33) + at unknown (http://localhost:3000/node_modules/.vite/deps/chunk-6FRHHHSJ.js?v=d328572b:2601:23)`, + foundInSpec: + 'e2e/playwright/testing-selections.spec.ts Solids should be select and deletable', + project: 'webkit', + }, + { + name: 'Unhandled Promise Rejection', + message: 'ReferenceError: Cannot access uninitialized variable.', + stack: `Unhandled Promise Rejection: ReferenceError: Cannot access uninitialized variable. + at setDiagnosticsForCurrentErrors (http://localhost:3000/src/lang/KclSingleton.ts:90:18) + at kclErrors (http://localhost:3000/src/lang/KclSingleton.ts:82:40) + at safeParse (http://localhost:3000/src/lang/KclSingleton.ts:150:9) + at unknown (http://localhost:3000/src/lang/KclSingleton.ts:113:32)`, + foundInSpec: + 'e2e/playwright/testing-segment-overlays.spec.ts angledLineToX', + project: 'webkit', + }, + { + name: 'Unhandled Promise Rejection', + message: 'sketchGroup not found', + stack: `Unhandled Promise Rejection: sketchGroup not found + at unknown (http://localhost:3000/src/machines/modelingMachine.ts:911:49)`, + foundInSpec: + 'e2e/playwright/testing-selections.spec.ts Deselecting line tool should mean nothing happens on click', + project: 'webkit', + }, + { + name: 'Unhandled Promise Rejection', + message: + 'engine error: [{"error_code":"bad_request","message":"Cannot set the camera position with these values"}]', + stack: + 'Unhandled Promise Rejection: engine error: [{"error_code":"bad_request","message":"Cannot set the camera position with these values"}]', + foundInSpec: + 'e2e/playwright/testing-camera-movement.spec.ts Zoom should be consistent when exiting or entering sketches', + project: 'webkit', + }, + { + name: 'SecurityError', + stack: `SecurityError: Failed to read the 'localStorage' property from 'Window': Access is denied for this document. + at :13:5 + at :18:5 + at :19:7`, + message: `Failed to read the 'localStorage' property from 'Window': Access is denied for this document.`, + project: 'Google Chrome', + foundInSpec: 'e2e/playwright/basic-sketch.spec.ts', + }, + { + name: ' - internal_engine', + stack: ` +`, + message: `Nothing to export`, + project: 'Google Chrome', + foundInSpec: 'e2e/playwright/regression-tests.spec.ts', + }, + { + name: 'SyntaxError', + stack: `SyntaxError: Unexpected end of JSON input + at crossPlatformFetch (http://localhost:3000/src/lib/crossPlatformFetch.ts:34:31) + at async sendTelemetry (http://localhost:3000/src/lib/textToCad.ts:179:3)`, + message: `Unexpected end of JSON input`, + project: 'Google Chrome', + foundInSpec: 'e2e/playwright/text-to-cad-tests.spec.ts', + }, + { + name: '{"kind"', + stack: ``, + message: `engine","sourceRanges":[[0,0]],"msg":"Failed to wait for promise from engine: JsValue(\\"Force interrupt, executionIsStale, new AST requested\\")"}`, + project: 'Google Chrome', + foundInSpec: 'e2e/playwright/testing-settings.spec.ts', + }, + ] + + const cleanString = (str: string) => str.replace(/[`"]/g, '') + const foundItem = whitelist.find( + (item) => + cleanString(exception.name) === cleanString(item.name) && + cleanString(exception.message).includes(cleanString(item.message)) + ) + + return foundItem !== undefined +} diff --git a/e2e/playwright/regression-tests.spec.ts b/e2e/playwright/regression-tests.spec.ts index 74ad8f7fac..369ca6f26a 100644 --- a/e2e/playwright/regression-tests.spec.ts +++ b/e2e/playwright/regression-tests.spec.ts @@ -11,8 +11,8 @@ import { import { TEST_CODE_TRIGGER_ENGINE_EXPORT_ERROR } from './storageStates' import { bracket } from 'lib/exampleKcl' -test.beforeEach(async ({ context, page }) => { - await setup(context, page) +test.beforeEach(async ({ context, page }, testInfo) => { + await setup(context, page, testInfo) }) test.afterEach(async ({ page }, testInfo) => { diff --git a/e2e/playwright/sketch-tests.spec.ts b/e2e/playwright/sketch-tests.spec.ts index 0d15c7dd14..e688a59f0d 100644 --- a/e2e/playwright/sketch-tests.spec.ts +++ b/e2e/playwright/sketch-tests.spec.ts @@ -9,8 +9,8 @@ import { } from './test-utils' import { uuidv4, roundOff } from 'lib/utils' -test.beforeEach(async ({ context, page }) => { - await setup(context, page) +test.beforeEach(async ({ context, page }, testInfo) => { + await setup(context, page, testInfo) }) test.afterEach(async ({ page }, testInfo) => { diff --git a/e2e/playwright/test-network-and-connection-issues.spec.ts b/e2e/playwright/test-network-and-connection-issues.spec.ts index c3bb8610af..ecea9a611e 100644 --- a/e2e/playwright/test-network-and-connection-issues.spec.ts +++ b/e2e/playwright/test-network-and-connection-issues.spec.ts @@ -2,8 +2,8 @@ import { test, expect } from '@playwright/test' import { commonPoints, getUtils, setup, tearDown } from './test-utils' -test.beforeEach(async ({ context, page }) => { - await setup(context, page) +test.beforeEach(async ({ context, page }, testInfo) => { + await setup(context, page, testInfo) }) test.afterEach(async ({ page }, testInfo) => { diff --git a/e2e/playwright/test-utils.ts b/e2e/playwright/test-utils.ts index ead587da59..81f3d86f44 100644 --- a/e2e/playwright/test-utils.ts +++ b/e2e/playwright/test-utils.ts @@ -26,6 +26,7 @@ import { import * as TOML from '@iarna/toml' import { SaveSettingsPayload } from 'lib/settings/settingsTypes' import { SETTINGS_FILE_NAME } from 'lib/constants' +import { isErrorWhitelisted } from './lib/console-error-whitelist' import { isArray } from 'lib/utils' import { reportRejection } from 'lib/trap' @@ -822,7 +823,11 @@ export async function tearDown(page: Page, testInfo: TestInfo) { // settingsOverrides may need to be augmented to take more generic items, // but we'll be strict for now -export async function setup(context: BrowserContext, page: Page) { +export async function setup( + context: BrowserContext, + page: Page, + testInfo?: TestInfo +) { await context.addInitScript( async ({ token, settingsKey, settings, IS_PLAYWRIGHT_KEY }) => { localStorage.clear() @@ -858,6 +863,8 @@ export async function setup(context: BrowserContext, page: Page) { secure: true, }, ]) + + failOnConsoleErrors(page, testInfo) // kill animations, speeds up tests and reduced flakiness await page.emulateMedia({ reducedMotion: 'reduce' }) @@ -931,6 +938,48 @@ export async function setupElectron({ return { electronApp, page, dir: projectDirName } } +function failOnConsoleErrors(page: Page, testInfo?: TestInfo) { + // enabled for chrome for now + if (page.context().browser()?.browserType().name() === 'chromium') { + page.on('pageerror', (exception) => { + if (isErrorWhitelisted(exception)) { + return + } + + // only set this env var to false if you want to collect console errors + // This can be configured in the GH workflow. This should be set to true by default (we want tests to fail when + // unwhitelisted console errors are detected). + if (process.env.FAIL_ON_CONSOLE_ERRORS === 'true') { + // Fail when running on CI and FAIL_ON_CONSOLE_ERRORS is set + // use expect to prevent page from closing and not cleaning up + expect(`An error was detected in the console: \r\n message:${exception.message} \r\n name:${exception.name} \r\n stack:${exception.stack} + + *Either fix the console error or add it to the whitelist defined in ./lib/console-error-whitelist.ts (if the error can be safely ignored) + `).toEqual('Console error detected') + } else { + // the (test-results/exceptions.txt) file will be uploaded as part of an upload artifact in GH + fsp + .appendFile( + './test-results/exceptions.txt', + [ + '~~~', + `triggered_by_test:${ + testInfo?.file + ' ' + (testInfo?.title || ' ') + }`, + `name:${exception.name}`, + `message:${exception.message}`, + `stack:${exception.stack}`, + `project:${testInfo?.project.name}`, + '~~~', + ].join('\n') + ) + .catch((err) => { + console.error(err) + }) + } + }) + } +} export async function isOutOfViewInScrollContainer( element: Locator, container: Locator diff --git a/e2e/playwright/testing-camera-movement.spec.ts b/e2e/playwright/testing-camera-movement.spec.ts index a43d8dad27..42824763d7 100644 --- a/e2e/playwright/testing-camera-movement.spec.ts +++ b/e2e/playwright/testing-camera-movement.spec.ts @@ -3,8 +3,8 @@ import { EngineCommand } from 'lang/std/artifactGraph' import { uuidv4 } from 'lib/utils' import { getUtils, setup, tearDown } from './test-utils' -test.beforeEach(async ({ context, page }) => { - await setup(context, page) +test.beforeEach(async ({ context, page }, testInfo) => { + await setup(context, page, testInfo) }) test.afterEach(async ({ page }, testInfo) => { diff --git a/e2e/playwright/testing-constraints.spec.ts b/e2e/playwright/testing-constraints.spec.ts index 69bbc85b20..ead09d76aa 100644 --- a/e2e/playwright/testing-constraints.spec.ts +++ b/e2e/playwright/testing-constraints.spec.ts @@ -3,8 +3,8 @@ import { test, expect } from '@playwright/test' import { getUtils, setup, tearDown, TEST_COLORS } from './test-utils' import { XOR } from 'lib/utils' -test.beforeEach(async ({ context, page }) => { - await setup(context, page) +test.beforeEach(async ({ context, page }, testInfo) => { + await setup(context, page, testInfo) }) test.afterEach(async ({ page }, testInfo) => { diff --git a/e2e/playwright/testing-gizmo.spec.ts b/e2e/playwright/testing-gizmo.spec.ts index 73046a27dc..88fa02304d 100644 --- a/e2e/playwright/testing-gizmo.spec.ts +++ b/e2e/playwright/testing-gizmo.spec.ts @@ -4,8 +4,8 @@ import { getUtils, setup, tearDown } from './test-utils' import { uuidv4 } from 'lib/utils' import { TEST_CODE_GIZMO } from './storageStates' -test.beforeEach(async ({ context, page }) => { - await setup(context, page) +test.beforeEach(async ({ context, page }, testInfo) => { + await setup(context, page, testInfo) }) test.afterEach(async ({ page }, testInfo) => { diff --git a/e2e/playwright/testing-segment-overlays.spec.ts b/e2e/playwright/testing-segment-overlays.spec.ts index 2a08b2a9cf..7cf7622f53 100644 --- a/e2e/playwright/testing-segment-overlays.spec.ts +++ b/e2e/playwright/testing-segment-overlays.spec.ts @@ -4,8 +4,8 @@ import { deg, getUtils, setup, tearDown, wiggleMove } from './test-utils' import { LineInputsType } from 'lang/std/sketchcombos' import { uuidv4 } from 'lib/utils' -test.beforeEach(async ({ context, page }) => { - await setup(context, page) +test.beforeEach(async ({ context, page }, testInfo) => { + await setup(context, page, testInfo) }) test.afterEach(async ({ page }, testInfo) => { diff --git a/e2e/playwright/testing-selections.spec.ts b/e2e/playwright/testing-selections.spec.ts index 7d7b245b5a..9dcf18993c 100644 --- a/e2e/playwright/testing-selections.spec.ts +++ b/e2e/playwright/testing-selections.spec.ts @@ -5,8 +5,8 @@ import { Coords2d } from 'lang/std/sketch' import { KCL_DEFAULT_LENGTH } from 'lib/constants' import { uuidv4 } from 'lib/utils' -test.beforeEach(async ({ context, page }) => { - await setup(context, page) +test.beforeEach(async ({ context, page }, testInfo) => { + await setup(context, page, testInfo) }) test.afterEach(async ({ page }, testInfo) => { diff --git a/e2e/playwright/testing-settings.spec.ts b/e2e/playwright/testing-settings.spec.ts index 4d09b096f2..b0f7eaa96e 100644 --- a/e2e/playwright/testing-settings.spec.ts +++ b/e2e/playwright/testing-settings.spec.ts @@ -16,8 +16,8 @@ import { } from './storageStates' import * as TOML from '@iarna/toml' -test.beforeEach(async ({ context, page }) => { - await setup(context, page) +test.beforeEach(async ({ context, page }, testInfo) => { + await setup(context, page, testInfo) }) test.afterEach(async ({ page }, testInfo) => { diff --git a/e2e/playwright/various.spec.ts b/e2e/playwright/various.spec.ts index fb947eca6a..f9acb93269 100644 --- a/e2e/playwright/various.spec.ts +++ b/e2e/playwright/various.spec.ts @@ -2,8 +2,8 @@ import { test, expect } from '@playwright/test' import { doExport, getUtils, makeTemplate, setup, tearDown } from './test-utils' -test.beforeEach(async ({ context, page }) => { - await setup(context, page) +test.beforeEach(async ({ context, page }, testInfo) => { + await setup(context, page, testInfo) }) test.afterEach(async ({ page }, testInfo) => { diff --git a/playwright.config.ts b/playwright.config.ts index 8813281b68..56f61628a1 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -57,10 +57,10 @@ export default defineConfig({ }, }, // or 'chrome-beta' }, - { - name: 'webkit', - use: { ...devices['Desktop Safari'] }, - }, + // { + // name: 'webkit', + // use: { ...devices['Desktop Safari'] }, + // }, // { // name: 'firefox', // use: { ...devices['Desktop Firefox'] },