WIP - seems to work except for asserting the logged telemetry events

This might be because the global `loggedEvents` is written to by several parallel tests at once. I'm not sure. I would have thought that each test case would be run in a separate process, but that does not necessarily seem to be the case.

vscode/playwright.config.ts

@@ -9,6 +9,6 @@ export default defineConfig({
     testDir: 'test/e2e',
     timeout: isWin ? 30000 : 20000,
     expect: {
-        timeout: isWin ? 5000 : 3000,
+        timeout: isWin ? 5000 : 6000,
     },
 })

vscode/src/configuration.ts

@@ -192,7 +192,11 @@ export const getFullConfig = async (): Promise<ConfigurationWithAccessToken> =>
     const isTesting = process.env.CODY_TESTING === 'true'
     const serverEndpoint =
         localStorage?.getEndpoint() ||
-        (isTesting ? `http://localhost:4930${process.env.VITEST_POOL_ID || 0}/` : DOTCOM_URL.href)
+        (isTesting
+            ? `http://localhost:4930${
+                  process.env.VITEST_POOL_ID ?? process.env.TEST_PARALLEL_INDEX ?? 0
+              }/`
+            : DOTCOM_URL.href)
     const accessToken = (await getAccessToken()) || null
     return { ...config, accessToken, serverEndpoint }
 }

vscode/test/e2e/chat-atFile.test.ts

@@ -16,10 +16,6 @@ test.extend<ExpectedEvents>({
         'CodyVSCodeExtension:at-mention:file:executed',
     ],
 })('@-mention file in chat', async ({ page, sidebar }) => {
-    // This test requires that the window be focused in the OS window manager because it deals with
-    // focus.
-    await page.bringToFront()
-
     await sidebarSignin(page, sidebar)
 
     await page.getByRole('button', { name: 'New Chat', exact: true }).click()

vscode/test/e2e/chat-input.test.ts

@@ -45,10 +45,6 @@ test.extend<ExpectedEvents>({
 })
 
 test('chat input focus', async ({ page, sidebar }) => {
-    // This test requires that the window be focused in the OS window manager because it deals with
-    // focus.
-    await page.bringToFront()
-
     await sidebarSignin(page, sidebar)
     // Open the buzz.ts file from the tree view,
     // and then submit a chat question from the command menu.

vscode/test/e2e/context-settings.test.ts

@@ -15,17 +15,23 @@ test.extend<ExpectedEvents>({
         'CodyVSCodeExtension:Auth:connected',
         'CodyVSCodeExtension:useEnhancedContextToggler:clicked',
     ],
-})('enhanced context selector is keyboard accessible', async ({ page, sidebar }) => {
-    // This test requires that the window be focused in the OS window manager because it deals with
-    // focus.
-    await page.bringToFront()
+})('enhanced context selector is keyboard accessible', async ({ page, sidebar }, testInfo) => {
+    // This test requires that the window be focused in the OS's window manager. This
+    // does not work when other tests are running in parallel. TODO(sqs): make this testable in parallel.
+    const isParallelTest = process.env.TEST_PARALLEL_INDEX !== undefined
+    if (isParallelTest) {
+        testInfo.skip()
+        return
+    }
 
+    await page.bringToFront()
     await sidebarSignin(page, sidebar)
     const chatFrame = await newChat(page)
     const contextSettingsButton = chatFrame.getByTitle('Configure Enhanced Context')
     await contextSettingsButton.focus()
 
     await page.keyboard.press('Space')
+
     // Opening the enhanced context settings should focus the checkbox for toggling it.
     const enhancedContextCheckbox = chatFrame.locator('#enhanced-context-checkbox')
     await expect(enhancedContextCheckbox).toBeFocused()

vscode/test/e2e/helpers.ts

@@ -167,13 +167,15 @@ export const test = base
 
             // Critical test to prevent event logging regressions.
             // Do not remove without consulting data analytics team.
-            try {
-                await assertEvents(loggedEvents, expectedEvents)
-            } catch (error) {
-                console.error('Expected events do not match actual events!')
-                console.log('Expected:', expectedEvents)
-                console.log('Logged:', loggedEvents)
-                throw error
+            if (testInfo.expectedStatus !== 'skipped') {
+                try {
+                    await assertEvents(loggedEvents, expectedEvents)
+                } catch (error) {
+                    console.error('Expected events do not match actual events!')
+                    console.log('Expected:', expectedEvents)
+                    console.log('Logged:', loggedEvents)
+                    throw error // TODO(sqs)
+                }
             }
             resetLoggedEvents()
 
@@ -258,7 +260,9 @@ async function buildWorkSpaceSettings(
     extraSettings: WorkspaceSettings
 ): Promise<void> {
     const settings = {
-        'cody.serverEndpoint': `http://localhost:4930${process.env.VITEST_POOL_ID || 0}`,
+        'cody.serverEndpoint': `http://localhost:4930${
+            process.env.VITEST_POOL_ID ?? process.env.TEST_PARALLEL_INDEX ?? 0
+        }`,
         'cody.commandCodeLenses': true,
         'cody.editorTitleCommandIcon': true,
         ...extraSettings,
@@ -298,7 +302,12 @@ export async function executeCommandInPalette(page: Page, commandName: string):
  * Verifies that loggedEvents contain all of expectedEvents (in any order).
  */
 export async function assertEvents(loggedEvents: string[], expectedEvents: string[]): Promise<void> {
-    await expect.poll(() => loggedEvents).toEqual(expect.arrayContaining(expectedEvents))
+    await expect
+        .poll(() => {
+            console.log('XX', loggedEvents)
+            return loggedEvents
+        })
+        .toEqual(expect.arrayContaining(expectedEvents))
 }
 
 // Creates a temporary directory, calls `f`, and then deletes the temporary

vscode/test/fixtures/mock-server.ts

@@ -18,7 +18,7 @@ interface MockRequest {
     }
 }
 
-const SERVER_PORT = Number(`4930${process.env.VITEST_POOL_ID || 0}`)
+const SERVER_PORT = Number(`4930${process.env.VITEST_POOL_ID ?? process.env.TEST_PARALLEL_INDEX ?? 0}`)
 export const SERVER_URL = `http://localhost:${SERVER_PORT}`
 export const VALID_TOKEN = 'sgp_1234567890123456789012345678901234567890'