Skip to content

Commit

Permalink
Move events into mockServer
Browse files Browse the repository at this point in the history
  • Loading branch information
@toolmantim
toolmantim committed Mar 18, 2024
1 parent 8200f48 commit 05cc633
Show file tree
Hide file tree
Showing 4 changed files with 140 additions and 40 deletions.
4 changes: 0 additions & 4 deletions vscode/test/e2e/command-custom.test.ts
View file
Expand Up @@ -12,10 +12,6 @@ import { testGitWorkspace } from './utils/gitWorkspace'

const test = baseTest.extend<DotcomUrlOverride>({ dotcomUrl: mockServer.SERVER_URL })

test.beforeEach(() => {
mockServer.resetLoggedEvents()
})

test.extend<ExpectedEvents>({
// list of events we expect this test to log, add to this list as needed
expectedEvents: [
Expand Down
18 changes: 9 additions & 9 deletions vscode/test/e2e/command-menu.test.ts
View file
Expand Up @@ -2,11 +2,18 @@ import { expect } from '@playwright/test'

import * as mockServer from '../fixtures/mock-server'
import { sidebarExplorer, sidebarSignin } from './common'
import { type DotcomUrlOverride, assertEvents, test as baseTest } from './helpers'
import { type DotcomUrlOverride, type ExpectedEvents, test as baseTest } from './helpers'

const test = baseTest.extend<DotcomUrlOverride>({ dotcomUrl: mockServer.SERVER_URL })

test('Start a new chat from Cody Command Menu', async ({ page, sidebar }) => {
test.extend<ExpectedEvents>({
// list of events we expect this test to log, add to this list as needed
expectedEvents: [
'CodyVSCodeExtension:menu:command:default:clicked',
'CodyVSCodeExtension:chat-question:submitted',
'CodyVSCodeExtension:chat-question:executed',
],
})('Start a new chat from Cody Command Menu', async ({ page, sidebar }) => {
// Sign into Cody
await sidebarSignin(page, sidebar)

Expand Down Expand Up @@ -34,11 +41,4 @@ test('Start a new chat from Cody Command Menu', async ({ page, sidebar }) => {
// the question should show up in the chat panel on submit
const chatPanel = page.frameLocator('iframe.webview').last().frameLocator('iframe')
await chatPanel.getByText('hello from the assistant').hover()

const expectedEvents = [
'CodyVSCodeExtension:menu:command:default:clicked',
'CodyVSCodeExtension:chat-question:submitted',
'CodyVSCodeExtension:chat-question:executed',
]
await assertEvents(mockServer.loggedEvents, expectedEvents)
})
127 changes: 122 additions & 5 deletions vscode/test/e2e/helpers.ts
View file
Expand Up @@ -7,7 +7,7 @@ import { type Frame, type FrameLocator, type Page, expect, test as base } from '
import { _electron as electron } from 'playwright'
import * as uuid from 'uuid'

import { MockServer, loggedEvents, resetLoggedEvents, sendTestInfo } from '../fixtures/mock-server'
import { MockServer, sendTestInfo } from '../fixtures/mock-server'

import { installVsCode } from './install-deps'
import { buildCustomCommandConfigFile } from './utils/buildCustomCommands'
Expand Down Expand Up @@ -85,7 +85,7 @@ export const test = base
workspaceDirectory,
extraWorkspaceSettings,
dotcomUrl,
server: MockServer,
server,
expectedEvents,
},
use,
Expand Down Expand Up @@ -169,15 +169,17 @@ export const test = base
// Do not remove without consulting data analytics team.
if (testInfo.expectedStatus !== 'skipped') {
try {
await assertEvents(loggedEvents, expectedEvents)
console.log(
`Process pid ${process.pid}, test id ${testInfo.testId}, title ${testInfo.title}`
)
await assertEvents(server.loggedEvents, expectedEvents)
} catch (error) {
console.error('Expected events do not match actual events!')
console.log('Expected:', expectedEvents)
console.log('Logged:', loggedEvents)
console.log('Logged:', server.loggedEvents)
throw error // TODO(sqs)
}
}
resetLoggedEvents()

await app.close()

Expand All @@ -190,6 +192,121 @@ export const test = base
await rmSyncWithRetries(extensionsDirectory, { recursive: true })
},
})
.extend({
page: async (
{
page: _page,
workspaceDirectory,
extraWorkspaceSettings,
dotcomUrl,
server,
expectedEvents,
},
use,
testInfo
) => {
void _page

const vscodeRoot = path.resolve(__dirname, '..', '..')

const vscodeExecutablePath = await installVsCode()
const extensionDevelopmentPath = vscodeRoot

const userDataDirectory = mkdtempSync(path.join(os.tmpdir(), 'cody-vsce'))
const extensionsDirectory = mkdtempSync(path.join(os.tmpdir(), 'cody-vsce'))
const videoDirectory = path.join(
vscodeRoot,
'..',
'playwright',
escapeToPath(testInfo.title)
)

await buildWorkSpaceSettings(workspaceDirectory, extraWorkspaceSettings)
await buildCustomCommandConfigFile(workspaceDirectory)

sendTestInfo(testInfo.title, testInfo.testId, uuid.v4())

let dotcomUrlOverride: { [key: string]: string } = {}
if (dotcomUrl) {
dotcomUrlOverride = { TESTING_DOTCOM_URL: dotcomUrl }
}

// See: https://github.com/microsoft/vscode-test/blob/main/lib/runTest.ts
const app = await electron.launch({
executablePath: vscodeExecutablePath,
env: {
...process.env,
...dotcomUrlOverride,
CODY_TESTING: 'true',
},
args: [
// https://github.com/microsoft/vscode/issues/84238
'--no-sandbox',
// https://github.com/microsoft/vscode-test/issues/120
'--disable-updates',
'--skip-welcome',
'--skip-release-notes',
'--disable-workspace-trust',
`--extensionDevelopmentPath=${extensionDevelopmentPath}`,
`--user-data-dir=${userDataDirectory}`,
`--extensions-dir=${extensionsDirectory}`,
workspaceDirectory,
],
recordVideo: {
dir: videoDirectory,
},
})

await waitUntil(() => app.windows().length > 0)

const page = await app.firstWindow()

// Bring the cody sidebar to the foreground if not already visible
if (!(await page.getByRole('heading', { name: 'Cody: Chat' }).isVisible())) {
await page.click('[aria-label="Cody"]')
}
// Ensure that we remove the hover from the activity icon
await page.getByRole('heading', { name: 'Cody: Chat' }).hover()
// Wait for Cody to become activated
// TODO(philipp-spiess): Figure out which playwright matcher we can use that works for
// the signed-in and signed-out cases
await new Promise(resolve => setTimeout(resolve, 500))

// Ensure we're signed out.
if (await page.isVisible('[aria-label="User Settings"]')) {
await signOut(page)
}

await use(page)

// Critical test to prevent event logging regressions.
// Do not remove without consulting data analytics team.
if (testInfo.expectedStatus !== 'skipped') {
try {
console.log(
`Process pid ${process.pid}, test id ${testInfo.testId}, title ${testInfo.title}`
)
await assertEvents(server.loggedEvents, expectedEvents)
} catch (error) {
console.error('Expected events do not match actual events!')
console.log('Expected:', expectedEvents)
console.log('Logged:', server.loggedEvents)
throw error // TODO(sqs)
}
}

await app.close()

// Delete the recorded video if the test passes
if (testInfo.status === 'passed') {
await rmSyncWithRetries(videoDirectory, { recursive: true })
}

await rmSyncWithRetries(userDataDirectory, { recursive: true })
await rmSyncWithRetries(extensionsDirectory, { recursive: true })
},
})

.extend<{ sidebar: Frame }>({
sidebar: async ({ page }, use) => {
const sidebar = await getCodySidebar(page)
Expand Down
31 changes: 9 additions & 22 deletions vscode/test/fixtures/mock-server.ts
View file
Expand Up @@ -141,6 +141,8 @@ class GraphQlMock {
// Lets the test change the behavior of the mock server.
export class MockServer {
graphQlMocks: Map<string, GraphQlMock> = new Map()
loggedEvents: string[] = []
loggedV2Events: string[] = []

constructor(public readonly express: express.Express) {}

Expand All @@ -163,7 +165,12 @@ export class MockServer {
// endpoint which will accept the data that you want to send in that you will add your pubsub code
app.post('/.api/testLogging', (req, res) => {
void logTestingData('legacy', req.body)
storeLoggedEvents(req.body)
interface ParsedEvent {
event: string
}
const parsedEvent = JSON.parse(JSON.stringify(req.body)) as ParsedEvent
const name = parsedEvent.event
controller.loggedEvents.push(name)
res.status(200)
})

Expand All @@ -177,7 +184,7 @@ export class MockServer {
'cody.extension', // extension setup events can behave differently in test environments
].includes(event.feature)
) {
loggedV2Events.push(`${event.feature}/${event.action}`)
controller.loggedV2Events.push(`${event.feature}/${event.action}`)
}
}
res.status(200)
Expand Down Expand Up @@ -516,23 +523,3 @@ export function sendTestInfo(testName: string, testID: string, testRunID: string
currentTestID = testID || ''
currentTestRunID = testRunID || ''
}

export let loggedEvents: string[] = []

// Events recorded using the new event recorders
// Needs to be recorded separately from the legacy events to ensure ordering
// is stable.
let loggedV2Events: string[] = []

export function resetLoggedEvents(): void {
loggedEvents = []
loggedV2Events = []
}
function storeLoggedEvents(event: string): void {
interface ParsedEvent {
event: string
}
const parsedEvent = JSON.parse(JSON.stringify(event)) as ParsedEvent
const name = parsedEvent.event
loggedEvents.push(name)
}

0 comments on commit 05cc633

Please sign in to comment.