diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml index 1e40d0dfd51..5a75040866b 100644 --- a/.github/workflows/end-to-end-tests.yaml +++ b/.github/workflows/end-to-end-tests.yaml @@ -130,8 +130,12 @@ jobs: if: steps.playwright-cache.outputs.cache-hit != 'true' run: yarn playwright install --with-deps --no-shell chromium + # We skip tests tagged with @mergequeue when running on PRs, but run them in MQ and everywhere else - name: Run Playwright tests - run: yarn playwright test --shard ${{ matrix.runner }}/${{ strategy.job-total }} + run: | + yarn playwright test \ + --shard "${{ matrix.runner }}/${{ strategy.job-total }}" \ + ${{ github.event_name == 'pull_request' && '--grep-invert @mergequeue' || '' }} - name: Upload blob report to GitHub Actions Artifacts if: always() diff --git a/docs/playwright.md b/docs/playwright.md index 1454c4868fe..4af3194220a 100644 --- a/docs/playwright.md +++ b/docs/playwright.md @@ -217,3 +217,10 @@ instead of the native `toHaveScreenshot`. If you are running Linux and are unfortunate that the screenshots are not rendering identically, you may wish to specify `--ignore-snapshots` and rely on Docker to render them for you. + +## Test Tags + +We use test tags to categorise tests for running subsets more efficiently. + +- `@mergequeue`: Tests that are slow or flaky and cover areas of the app we update seldom, should not be run on every PR commit but will be run in the Merge Queue. +- `@screenshot`: Tests that use `toMatchScreenshot` to speed up a run of `test:playwright:screenshots`. A test with this tag must not also have the `@mergequeue` tag as this would cause false positives in the stale screenshot detection. diff --git a/playwright/e2e/read-receipts/editing-messages-in-threads.spec.ts b/playwright/e2e/read-receipts/editing-messages-in-threads.spec.ts index 277de628769..42191831c80 100644 --- a/playwright/e2e/read-receipts/editing-messages-in-threads.spec.ts +++ b/playwright/e2e/read-receipts/editing-messages-in-threads.spec.ts @@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details. import { test } from "."; -test.describe("Read receipts", () => { +test.describe("Read receipts", { tag: "@mergequeue" }, () => { test.describe("editing messages", () => { test.describe("in threads", () => { test("An edit of a threaded message makes the room unread", async ({ diff --git a/playwright/e2e/read-receipts/editing-messages-main-timeline.spec.ts b/playwright/e2e/read-receipts/editing-messages-main-timeline.spec.ts index 027ab08e2d2..a4648223050 100644 --- a/playwright/e2e/read-receipts/editing-messages-main-timeline.spec.ts +++ b/playwright/e2e/read-receipts/editing-messages-main-timeline.spec.ts @@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details. import { test } from "."; -test.describe("Read receipts", () => { +test.describe("Read receipts", { tag: "@mergequeue" }, () => { test.describe("editing messages", () => { test.describe("in the main timeline", () => { test("Editing a message leaves a room read", async ({ roomAlpha: room1, roomBeta: room2, util, msg }) => { diff --git a/playwright/e2e/read-receipts/editing-messages-thread-roots.spec.ts b/playwright/e2e/read-receipts/editing-messages-thread-roots.spec.ts index e653b5d9bd6..506ed603bda 100644 --- a/playwright/e2e/read-receipts/editing-messages-thread-roots.spec.ts +++ b/playwright/e2e/read-receipts/editing-messages-thread-roots.spec.ts @@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details. import { test } from "."; -test.describe("Read receipts", () => { +test.describe("Read receipts", { tag: "@mergequeue" }, () => { test.describe("editing messages", () => { test.describe("thread roots", () => { test("An edit of a thread root leaves the room read", async ({ diff --git a/playwright/e2e/read-receipts/high-level.spec.ts b/playwright/e2e/read-receipts/high-level.spec.ts index 7d4f4eb1330..457cf994814 100644 --- a/playwright/e2e/read-receipts/high-level.spec.ts +++ b/playwright/e2e/read-receipts/high-level.spec.ts @@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details. import { customEvent, many, test } from "."; -test.describe("Read receipts", () => { +test.describe("Read receipts", { tag: "@mergequeue" }, () => { test.describe("Ignored events", () => { test("If all events after receipt are unimportant, the room is read", async ({ roomAlpha: room1, diff --git a/playwright/e2e/read-receipts/message-ordering.spec.ts b/playwright/e2e/read-receipts/message-ordering.spec.ts index 65875cf4a9e..d7f77fae5f5 100644 --- a/playwright/e2e/read-receipts/message-ordering.spec.ts +++ b/playwright/e2e/read-receipts/message-ordering.spec.ts @@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details. import { test } from "."; -test.describe("Read receipts", () => { +test.describe("Read receipts", { tag: "@mergequeue" }, () => { test.describe("Message ordering", () => { test.describe("in the main timeline", () => { test.fixme( diff --git a/playwright/e2e/read-receipts/missing-referents.spec.ts b/playwright/e2e/read-receipts/missing-referents.spec.ts index f798d7d4552..a1741851e2c 100644 --- a/playwright/e2e/read-receipts/missing-referents.spec.ts +++ b/playwright/e2e/read-receipts/missing-referents.spec.ts @@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details. import { test } from "."; -test.describe("Read receipts", () => { +test.describe("Read receipts", { tag: "@mergequeue" }, () => { test.describe("messages with missing referents", () => { test.fixme( "A message in an unknown thread is not visible and the room is read", diff --git a/playwright/e2e/read-receipts/new-messages-in-threads.spec.ts b/playwright/e2e/read-receipts/new-messages-in-threads.spec.ts index 91d850fac89..5407f3cb44a 100644 --- a/playwright/e2e/read-receipts/new-messages-in-threads.spec.ts +++ b/playwright/e2e/read-receipts/new-messages-in-threads.spec.ts @@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details. import { many, test } from "."; -test.describe("Read receipts", () => { +test.describe("Read receipts", { tag: "@mergequeue" }, () => { test.describe("new messages", () => { test.describe("in threads", () => { test("Receiving a message makes a room unread", async ({ diff --git a/playwright/e2e/read-receipts/new-messages-main-timeline.spec.ts b/playwright/e2e/read-receipts/new-messages-main-timeline.spec.ts index 2000e444d64..92f7b10cdd7 100644 --- a/playwright/e2e/read-receipts/new-messages-main-timeline.spec.ts +++ b/playwright/e2e/read-receipts/new-messages-main-timeline.spec.ts @@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details. import { many, test } from "."; -test.describe("Read receipts", () => { +test.describe("Read receipts", { tag: "@mergequeue" }, () => { test.describe("new messages", () => { test.describe("in the main timeline", () => { test("Receiving a message makes a room unread", async ({ diff --git a/playwright/e2e/read-receipts/new-messages-thread-roots.spec.ts b/playwright/e2e/read-receipts/new-messages-thread-roots.spec.ts index 878d0d4419f..3c8ed7849f4 100644 --- a/playwright/e2e/read-receipts/new-messages-thread-roots.spec.ts +++ b/playwright/e2e/read-receipts/new-messages-thread-roots.spec.ts @@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details. import { many, test } from "."; -test.describe("Read receipts", () => { +test.describe("Read receipts", { tag: "@mergequeue" }, () => { test.describe("new messages", () => { test.describe("thread roots", () => { test("Reading a thread root does not mark the thread as read", async ({ diff --git a/playwright/e2e/read-receipts/notifications.spec.ts b/playwright/e2e/read-receipts/notifications.spec.ts index 3050987be77..46edc9a7a32 100644 --- a/playwright/e2e/read-receipts/notifications.spec.ts +++ b/playwright/e2e/read-receipts/notifications.spec.ts @@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details. import { test } from "."; -test.describe("Read receipts", () => { +test.describe("Read receipts", { tag: "@mergequeue" }, () => { test.describe("Notifications", () => { test.describe("in the main timeline", () => { test.fixme("A new message that mentions me shows a notification", () => {}); diff --git a/playwright/e2e/read-receipts/reactions-in-threads.spec.ts b/playwright/e2e/read-receipts/reactions-in-threads.spec.ts index bc4a1847441..45b5e071ecc 100644 --- a/playwright/e2e/read-receipts/reactions-in-threads.spec.ts +++ b/playwright/e2e/read-receipts/reactions-in-threads.spec.ts @@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details. import { test, expect } from "."; -test.describe("Read receipts", () => { +test.describe("Read receipts", { tag: "@mergequeue" }, () => { test.describe("reactions", () => { test.describe("in threads", () => { test("A reaction to a threaded message does not make the room unread", async ({ diff --git a/playwright/e2e/read-receipts/reactions-main-timeline.spec.ts b/playwright/e2e/read-receipts/reactions-main-timeline.spec.ts index 59d6eaea400..16d5c92eca0 100644 --- a/playwright/e2e/read-receipts/reactions-main-timeline.spec.ts +++ b/playwright/e2e/read-receipts/reactions-main-timeline.spec.ts @@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details. import { test } from "."; -test.describe("Read receipts", () => { +test.describe("Read receipts", { tag: "@mergequeue" }, () => { test.describe("reactions", () => { test.describe("in the main timeline", () => { test("Receiving a reaction to a message does not make a room unread", async ({ diff --git a/playwright/e2e/read-receipts/reactions-thread-roots.spec.ts b/playwright/e2e/read-receipts/reactions-thread-roots.spec.ts index 219a73d5e46..817597a27e0 100644 --- a/playwright/e2e/read-receipts/reactions-thread-roots.spec.ts +++ b/playwright/e2e/read-receipts/reactions-thread-roots.spec.ts @@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details. import { test } from "."; -test.describe("Read receipts", () => { +test.describe("Read receipts", { tag: "@mergequeue" }, () => { test.describe("reactions", () => { test.describe("thread roots", () => { test("A reaction to a thread root does not make the room unread", async ({ diff --git a/playwright/e2e/read-receipts/read-receipts.spec.ts b/playwright/e2e/read-receipts/read-receipts.spec.ts index 3056cc4a54b..f6515361f23 100644 --- a/playwright/e2e/read-receipts/read-receipts.spec.ts +++ b/playwright/e2e/read-receipts/read-receipts.spec.ts @@ -13,7 +13,7 @@ import { ElementAppPage } from "../../pages/ElementAppPage"; import { Bot } from "../../pages/bot"; import { test } from "."; -test.describe("Read receipts", () => { +test.describe("Read receipts", { tag: "@mergequeue" }, () => { test.use({ displayName: "Mae", botCreateOpts: { displayName: "Other User" }, diff --git a/playwright/e2e/read-receipts/redactions-in-threads.spec.ts b/playwright/e2e/read-receipts/redactions-in-threads.spec.ts index 715bb4e9fc8..25c19a4f975 100644 --- a/playwright/e2e/read-receipts/redactions-in-threads.spec.ts +++ b/playwright/e2e/read-receipts/redactions-in-threads.spec.ts @@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details. import { test } from "."; -test.describe("Read receipts", () => { +test.describe("Read receipts", { tag: "@mergequeue" }, () => { test.describe("redactions", () => { test.describe("in threads", () => { test("Redacting the threaded message pointed to by my receipt leaves the room read", async ({ diff --git a/playwright/e2e/read-receipts/redactions-main-timeline.spec.ts b/playwright/e2e/read-receipts/redactions-main-timeline.spec.ts index c1dceda6a00..143d9685d8f 100644 --- a/playwright/e2e/read-receipts/redactions-main-timeline.spec.ts +++ b/playwright/e2e/read-receipts/redactions-main-timeline.spec.ts @@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details. import { test } from "."; -test.describe("Read receipts", () => { +test.describe("Read receipts", { tag: "@mergequeue" }, () => { test.describe("redactions", () => { test.describe("in the main timeline", () => { test("Redacting the message pointed to by my receipt leaves the room read", async ({ diff --git a/playwright/e2e/read-receipts/redactions-thread-roots.spec.ts b/playwright/e2e/read-receipts/redactions-thread-roots.spec.ts index a8dc38c47ee..01f296075c9 100644 --- a/playwright/e2e/read-receipts/redactions-thread-roots.spec.ts +++ b/playwright/e2e/read-receipts/redactions-thread-roots.spec.ts @@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details. import { test } from "."; -test.describe("Read receipts", () => { +test.describe("Read receipts", { tag: "@mergequeue" }, () => { test.describe("redactions", () => { test.describe("thread roots", () => { test("Redacting a thread root after it was read leaves the room read", async ({ diff --git a/playwright/e2e/read-receipts/room-list-order.spec.ts b/playwright/e2e/read-receipts/room-list-order.spec.ts index 052e2d756a3..80dda202a3a 100644 --- a/playwright/e2e/read-receipts/room-list-order.spec.ts +++ b/playwright/e2e/read-receipts/room-list-order.spec.ts @@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details. import { test } from "."; -test.describe("Read receipts", () => { +test.describe("Read receipts", { tag: "@mergequeue" }, () => { test.describe("Room list order", () => { test("Rooms with unread messages appear at the top of room list if 'unread first' is selected", async ({ roomAlpha: room1,