diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml index acd59406e9..97d9692a38 100644 --- a/.github/workflows/end-to-end-tests.yaml +++ b/.github/workflows/end-to-end-tests.yaml @@ -190,13 +190,14 @@ jobs: - name: Merge into HTML Report if: inputs.skip != true - run: yarn playwright merge-reports --reporter=html,./playwright/flaky-reporter.ts ./all-blob-reports + run: yarn playwright merge-reports --reporter=html,./playwright/flaky-reporter.ts,./playwright/stale-screenshot-reporter.ts ./all-blob-reports env: # Only pass creds to the flaky-reporter on main branch runs GITHUB_TOKEN: ${{ github.ref_name == 'develop' && secrets.ELEMENT_BOT_TOKEN || '' }} + # Upload the HTML report even if one of our reporters fails, this can happen when stale screenshots are detected - name: Upload HTML report - if: inputs.skip != true + if: always() && inputs.skip != true uses: actions/upload-artifact@v4 with: name: html-report diff --git a/package.json b/package.json index 14635fdeee..e483fb3948 100644 --- a/package.json +++ b/package.json @@ -213,6 +213,7 @@ "fake-indexeddb": "^6.0.0", "fetch-mock-jest": "^1.5.1", "fs-extra": "^11.0.0", + "glob": "^11.0.0", "jest": "^29.6.2", "jest-canvas-mock": "^2.5.2", "jest-environment-jsdom": "^29.6.2", @@ -223,6 +224,7 @@ "matrix-web-i18n": "^3.2.1", "mocha-junit-reporter": "^2.2.0", "node-fetch": "2", + "playwright-core": "^1.45.1", "postcss-scss": "^4.0.4", "prettier": "3.3.2", "raw-loader": "^4.0.2", diff --git a/playwright/@types/playwright-core.d.ts b/playwright/@types/playwright-core.d.ts new file mode 100644 index 0000000000..0ef2ca0ece --- /dev/null +++ b/playwright/@types/playwright-core.d.ts @@ -0,0 +1,20 @@ +/* +Copyright 2024 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +declare module "playwright-core/lib/utils" { + // This type is not public in playwright-core utils + export function sanitizeForFilePath(filePath: string): string; +} diff --git a/playwright/e2e/settings/general-room-settings-tab.spec.ts b/playwright/e2e/settings/general-room-settings-tab.spec.ts index ec3c14b2ca..123c214288 100644 --- a/playwright/e2e/settings/general-room-settings-tab.spec.ts +++ b/playwright/e2e/settings/general-room-settings-tab.spec.ts @@ -34,7 +34,7 @@ test.describe("General room settings tab", () => { // Assert that "Show less" details element is rendered await expect(settings.getByText("Show less")).toBeVisible(); - await expect(settings).toMatchScreenshot(); + await expect(settings).toMatchScreenshot("General-room-settings-tab-should-be-rendered-properly-1.png"); // Click the "Show less" details element await settings.getByText("Show less").click(); diff --git a/playwright/e2e/settings/preferences-user-settings-tab.spec.ts b/playwright/e2e/settings/preferences-user-settings-tab.spec.ts index 22baa19a8a..a67909b47b 100644 --- a/playwright/e2e/settings/preferences-user-settings-tab.spec.ts +++ b/playwright/e2e/settings/preferences-user-settings-tab.spec.ts @@ -31,7 +31,7 @@ test.describe("Preferences user settings tab", () => { // Assert that the top heading is rendered await expect(tab.getByRole("heading", { name: "Preferences" })).toBeVisible(); - await expect(tab).toMatchScreenshot(); + await expect(tab).toMatchScreenshot("Preferences-user-settings-tab-should-be-rendered-properly-1.png"); }); test("should be able to change the app language", async ({ uut, user }) => { diff --git a/playwright/e2e/settings/security-user-settings-tab.spec.ts b/playwright/e2e/settings/security-user-settings-tab.spec.ts index 381b291e97..8d1f442ba5 100644 --- a/playwright/e2e/settings/security-user-settings-tab.spec.ts +++ b/playwright/e2e/settings/security-user-settings-tab.spec.ts @@ -47,7 +47,9 @@ test.describe("Security user settings tab", () => { test("should be rendered properly", async ({ app, page }) => { const tab = await app.settings.openUserSettings("Security"); await tab.getByRole("button", { name: "Learn more" }).click(); - await expect(page.locator(".mx_AnalyticsLearnMoreDialog_wrapper .mx_Dialog")).toMatchScreenshot(); + await expect(page.locator(".mx_AnalyticsLearnMoreDialog_wrapper .mx_Dialog")).toMatchScreenshot( + "Security-user-settings-tab-with-posthog-enable-b5d89-csLearnMoreDialog-should-be-rendered-properly-1.png", + ); }); }); diff --git a/playwright/e2e/user-onboarding/user-onboarding-new.spec.ts b/playwright/e2e/user-onboarding/user-onboarding-new.spec.ts index 09a140d441..0799dc0f60 100644 --- a/playwright/e2e/user-onboarding/user-onboarding-new.spec.ts +++ b/playwright/e2e/user-onboarding/user-onboarding-new.spec.ts @@ -35,7 +35,9 @@ test.describe("User Onboarding (new user)", () => { }); test("page is shown and preference exists", async ({ page, app }) => { - await expect(page.locator(".mx_UserOnboardingPage")).toMatchScreenshot(); + await expect(page.locator(".mx_UserOnboardingPage")).toMatchScreenshot( + "User-Onboarding-new-user-page-is-shown-and-preference-exists-1.png", + ); await app.settings.openUserSettings("Preferences"); await expect(page.getByText("Show shortcut to welcome checklist above the room list")).toBeVisible(); }); diff --git a/playwright/element-web-test.ts b/playwright/element-web-test.ts index 3b12cba8cf..d6fd1b48c1 100644 --- a/playwright/element-web-test.ts +++ b/playwright/element-web-test.ts @@ -15,9 +15,10 @@ limitations under the License. */ import { test as base, expect as baseExpect, Locator, Page, ExpectMatcherState, ElementHandle } from "@playwright/test"; +import { sanitizeForFilePath } from "playwright-core/lib/utils"; import AxeBuilder from "@axe-core/playwright"; import _ from "lodash"; -import { basename } from "node:path"; +import { basename, extname } from "node:path"; import type mailhog from "mailhog"; import type { IConfigOptions } from "../src/IConfigOptions"; @@ -298,11 +299,18 @@ export const test = base.extend<{ }, }); +// Based on https://github.com/microsoft/playwright/blob/2b77ed4d7aafa85a600caa0b0d101b72c8437eeb/packages/playwright/src/util.ts#L206C8-L210C2 +function sanitizeFilePathBeforeExtension(filePath: string): string { + const ext = extname(filePath); + const base = filePath.substring(0, filePath.length - ext.length); + return sanitizeForFilePath(base) + ext; +} + export const expect = baseExpect.extend({ async toMatchScreenshot( this: ExpectMatcherState, receiver: Page | Locator, - name?: `${string}.png`, + name: `${string}.png`, options?: { mask?: Array; omitBackground?: boolean; @@ -311,6 +319,9 @@ export const expect = baseExpect.extend({ css?: string; }, ) { + const testInfo = test.info(); + if (!testInfo) throw new Error(`toMatchScreenshot() must be called during the test`); + const page = "page" in receiver ? receiver.page() : receiver; let hideTooltipsCss: string | undefined; @@ -354,9 +365,18 @@ export const expect = baseExpect.extend({ `, })) as ElementHandle; - await baseExpect(receiver).toHaveScreenshot(name, options); + const screenshotName = sanitizeFilePathBeforeExtension(name); + await baseExpect(receiver).toHaveScreenshot(screenshotName, options); await style.evaluate((tag) => tag.remove()); + + testInfo.annotations.push({ + // `_` prefix hides it from the HTML reporter + type: "_screenshot", + // include a path relative to `playwright/snapshots/` + description: testInfo.snapshotPath(screenshotName).split("/playwright/snapshots/", 2)[1], + }); + return { pass: true, message: () => "", name: "toMatchScreenshot" }; }, }); diff --git a/playwright/snapshots/settings/appearance-user-settings-tab.spec.ts/appearance-tab-linux.png b/playwright/snapshots/settings/appearance-user-settings-tab.spec.ts/appearance-tab-linux.png deleted file mode 100644 index c16b95d1de..0000000000 Binary files a/playwright/snapshots/settings/appearance-user-settings-tab.spec.ts/appearance-tab-linux.png and /dev/null differ diff --git a/playwright/snapshots/settings/appearance-user-settings-tab.spec.ts/font-slider-11-darwin.png b/playwright/snapshots/settings/appearance-user-settings-tab.spec.ts/font-slider-11-darwin.png deleted file mode 100644 index 12996f4e5b..0000000000 Binary files a/playwright/snapshots/settings/appearance-user-settings-tab.spec.ts/font-slider-11-darwin.png and /dev/null differ diff --git a/playwright/snapshots/settings/appearance-user-settings-tab.spec.ts/font-slider-11-linux.png b/playwright/snapshots/settings/appearance-user-settings-tab.spec.ts/font-slider-11-linux.png deleted file mode 100644 index f523146348..0000000000 Binary files a/playwright/snapshots/settings/appearance-user-settings-tab.spec.ts/font-slider-11-linux.png and /dev/null differ diff --git a/playwright/snapshots/settings/appearance-user-settings-tab.spec.ts/font-slider-21-darwin.png b/playwright/snapshots/settings/appearance-user-settings-tab.spec.ts/font-slider-21-darwin.png deleted file mode 100644 index 6b1e058c6a..0000000000 Binary files a/playwright/snapshots/settings/appearance-user-settings-tab.spec.ts/font-slider-21-darwin.png and /dev/null differ diff --git a/playwright/snapshots/settings/appearance-user-settings-tab.spec.ts/font-slider-21-linux.png b/playwright/snapshots/settings/appearance-user-settings-tab.spec.ts/font-slider-21-linux.png deleted file mode 100644 index 502e60cb1f..0000000000 Binary files a/playwright/snapshots/settings/appearance-user-settings-tab.spec.ts/font-slider-21-linux.png and /dev/null differ diff --git a/playwright/snapshots/settings/appearance-user-settings-tab.spec.ts/window-12px-linux.png b/playwright/snapshots/settings/appearance-user-settings-tab.spec.ts/window-12px-linux.png deleted file mode 100644 index 7d9400b0bd..0000000000 Binary files a/playwright/snapshots/settings/appearance-user-settings-tab.spec.ts/window-12px-linux.png and /dev/null differ diff --git a/playwright/snapshots/settings/appearance-user-settings-tab/appearance-user-settings-tab.spec.ts/message-layout-panel-bubble-linux.png b/playwright/snapshots/settings/appearance-user-settings-tab/appearance-user-settings-tab.spec.ts/message-layout-panel-bubble-linux.png deleted file mode 100644 index 3f39a3b01d..0000000000 Binary files a/playwright/snapshots/settings/appearance-user-settings-tab/appearance-user-settings-tab.spec.ts/message-layout-panel-bubble-linux.png and /dev/null differ diff --git a/playwright/snapshots/settings/appearance-user-settings-tab/appearance-user-settings-tab.spec.ts/message-layout-panel-modern-linux.png b/playwright/snapshots/settings/appearance-user-settings-tab/appearance-user-settings-tab.spec.ts/message-layout-panel-modern-linux.png deleted file mode 100644 index 74aaf9a763..0000000000 Binary files a/playwright/snapshots/settings/appearance-user-settings-tab/appearance-user-settings-tab.spec.ts/message-layout-panel-modern-linux.png and /dev/null differ diff --git a/playwright/snapshots/timeline/timeline.spec.ts/hidden-event-line-padding-modern-layout-linux.png b/playwright/snapshots/timeline/timeline.spec.ts/hidden-event-line-padding-modern-layout-linux.png deleted file mode 100644 index ceddab3312..0000000000 Binary files a/playwright/snapshots/timeline/timeline.spec.ts/hidden-event-line-padding-modern-layout-linux.png and /dev/null differ diff --git a/playwright/snapshots/timeline/timeline.spec.ts/hidden-event-line-zero-padding-irc-layout-linux.png b/playwright/snapshots/timeline/timeline.spec.ts/hidden-event-line-zero-padding-irc-layout-linux.png deleted file mode 100644 index 5fba124a92..0000000000 Binary files a/playwright/snapshots/timeline/timeline.spec.ts/hidden-event-line-zero-padding-irc-layout-linux.png and /dev/null differ diff --git a/playwright/snapshots/user-view/user-view.spec.ts/UserView-should-render-the-user-view-as-expected-1-linux.png b/playwright/snapshots/user-view/user-view.spec.ts/UserView-should-render-the-user-view-as-expected-1-linux.png deleted file mode 100644 index 75b64546d6..0000000000 Binary files a/playwright/snapshots/user-view/user-view.spec.ts/UserView-should-render-the-user-view-as-expected-1-linux.png and /dev/null differ diff --git a/playwright/stale-screenshot-reporter.ts b/playwright/stale-screenshot-reporter.ts new file mode 100644 index 0000000000..3a3d18e28c --- /dev/null +++ b/playwright/stale-screenshot-reporter.ts @@ -0,0 +1,74 @@ +/* +Copyright 2024 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/** + * Test reporter which compares the reported screenshots vs those on disk to find stale screenshots + * Only intended to run from within GitHub Actions + */ + +import path from "node:path"; +import { glob } from "glob"; + +import type { Reporter, TestCase } from "@playwright/test/reporter"; + +const snapshotRoot = path.join(__dirname, "snapshots"); + +class StaleScreenshotReporter implements Reporter { + private screenshots = new Set(); + private success = true; + + public onTestEnd(test: TestCase): void { + for (const annotation of test.annotations) { + if (annotation.type === "_screenshot") { + this.screenshots.add(annotation.description); + } + } + } + + private error(msg: string, file: string) { + if (process.env.GITHUB_ACTIONS) { + console.log(`::error file=${file}::${msg}`); + } + console.error(msg, file); + this.success = false; + } + + public async onExit(): Promise { + const screenshotFiles = new Set(await glob(`**/*.png`, { cwd: snapshotRoot })); + for (const screenshot of screenshotFiles) { + if (screenshot.split("-").at(-1) !== "linux.png") { + this.error( + "Found screenshot belonging to different platform, this should not be checked in", + screenshot, + ); + } + } + for (const screenshot of this.screenshots) { + screenshotFiles.delete(screenshot); + } + if (screenshotFiles.size > 0) { + for (const screenshot of screenshotFiles) { + this.error("Stale screenshot file", screenshot); + } + } + + if (!this.success) { + process.exit(1); + } + } +} + +export default StaleScreenshotReporter; diff --git a/yarn.lock b/yarn.lock index b6585e26ac..7aff84f1a0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7418,7 +7418,7 @@ pkg-dir@^4.2.0: dependencies: find-up "^4.0.0" -playwright-core@1.45.1: +playwright-core@1.45.1, playwright-core@^1.45.1: version "1.45.1" resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.45.1.tgz#549a2701556b58245cc75263f9fc2795c1158dc1" integrity sha512-LF4CUUtrUu2TCpDw4mcrAIuYrEjVDfT1cHbJMfwnE2+1b8PZcFzPNgvZCvq2JfQ4aTjRCCHw5EJ2tmr2NSzdPg==