Add mergequeue tag to read-receipts tests and skip running them on PR…

… commits (element-hq#28648) Signed-off-by: Michael Telatynski <[email protected]>
elecordapp · Dec 5, 2024 · 450cb60 · 450cb60
1 parent 7e03f38
commit 450cb60
Show file tree

Hide file tree

Showing 20 changed files with 30 additions and 19 deletions.
diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml
@@ -130,8 +130,12 @@ jobs:
               if: steps.playwright-cache.outputs.cache-hit != 'true'
               run: yarn playwright install --with-deps --no-shell chromium
 
+            # We skip tests tagged with @mergequeue when running on PRs, but run them in MQ and everywhere else
             - name: Run Playwright tests
-              run: yarn playwright test --shard ${{ matrix.runner }}/${{ strategy.job-total }}
+              run: |
+                  yarn playwright test \
+                      --shard "${{ matrix.runner }}/${{ strategy.job-total }}" \
+                      ${{ github.event_name == 'pull_request' && '--grep-invert @mergequeue' || '' }}
 
             - name: Upload blob report to GitHub Actions Artifacts
               if: always()

diff --git a/docs/playwright.md b/docs/playwright.md
@@ -217,3 +217,10 @@ instead of the native `toHaveScreenshot`.
 
 If you are running Linux and are unfortunate that the screenshots are not rendering identically,
 you may wish to specify `--ignore-snapshots` and rely on Docker to render them for you.
+
+## Test Tags
+
+We use test tags to categorise tests for running subsets more efficiently.
+
+- `@mergequeue`: Tests that are slow or flaky and cover areas of the app we update seldom, should not be run on every PR commit but will be run in the Merge Queue.
+- `@screenshot`: Tests that use `toMatchScreenshot` to speed up a run of `test:playwright:screenshots`. A test with this tag must not also have the `@mergequeue` tag as this would cause false positives in the stale screenshot detection.
diff --git a/playwright/e2e/read-receipts/editing-messages-in-threads.spec.ts b/playwright/e2e/read-receipts/editing-messages-in-threads.spec.ts
@@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details.
 
 import { test } from ".";
 
-test.describe("Read receipts", () => {
+test.describe("Read receipts", { tag: "@mergequeue" }, () => {
     test.describe("editing messages", () => {
         test.describe("in threads", () => {
             test("An edit of a threaded message makes the room unread", async ({

diff --git a/playwright/e2e/read-receipts/editing-messages-main-timeline.spec.ts b/playwright/e2e/read-receipts/editing-messages-main-timeline.spec.ts
@@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details.
 
 import { test } from ".";
 
-test.describe("Read receipts", () => {
+test.describe("Read receipts", { tag: "@mergequeue" }, () => {
     test.describe("editing messages", () => {
         test.describe("in the main timeline", () => {
             test("Editing a message leaves a room read", async ({ roomAlpha: room1, roomBeta: room2, util, msg }) => {

diff --git a/playwright/e2e/read-receipts/editing-messages-thread-roots.spec.ts b/playwright/e2e/read-receipts/editing-messages-thread-roots.spec.ts
@@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details.
 
 import { test } from ".";
 
-test.describe("Read receipts", () => {
+test.describe("Read receipts", { tag: "@mergequeue" }, () => {
     test.describe("editing messages", () => {
         test.describe("thread roots", () => {
             test("An edit of a thread root leaves the room read", async ({

diff --git a/playwright/e2e/read-receipts/high-level.spec.ts b/playwright/e2e/read-receipts/high-level.spec.ts
@@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details.
 
 import { customEvent, many, test } from ".";
 
-test.describe("Read receipts", () => {
+test.describe("Read receipts", { tag: "@mergequeue" }, () => {
     test.describe("Ignored events", () => {
         test("If all events after receipt are unimportant, the room is read", async ({
             roomAlpha: room1,

diff --git a/playwright/e2e/read-receipts/message-ordering.spec.ts b/playwright/e2e/read-receipts/message-ordering.spec.ts
@@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details.
 
 import { test } from ".";
 
-test.describe("Read receipts", () => {
+test.describe("Read receipts", { tag: "@mergequeue" }, () => {
     test.describe("Message ordering", () => {
         test.describe("in the main timeline", () => {
             test.fixme(

diff --git a/playwright/e2e/read-receipts/missing-referents.spec.ts b/playwright/e2e/read-receipts/missing-referents.spec.ts
@@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details.
 
 import { test } from ".";
 
-test.describe("Read receipts", () => {
+test.describe("Read receipts", { tag: "@mergequeue" }, () => {
     test.describe("messages with missing referents", () => {
         test.fixme(
             "A message in an unknown thread is not visible and the room is read",

diff --git a/playwright/e2e/read-receipts/new-messages-in-threads.spec.ts b/playwright/e2e/read-receipts/new-messages-in-threads.spec.ts
@@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details.
 
 import { many, test } from ".";
 
-test.describe("Read receipts", () => {
+test.describe("Read receipts", { tag: "@mergequeue" }, () => {
     test.describe("new messages", () => {
         test.describe("in threads", () => {
             test("Receiving a message makes a room unread", async ({

diff --git a/playwright/e2e/read-receipts/new-messages-main-timeline.spec.ts b/playwright/e2e/read-receipts/new-messages-main-timeline.spec.ts
@@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details.
 
 import { many, test } from ".";
 
-test.describe("Read receipts", () => {
+test.describe("Read receipts", { tag: "@mergequeue" }, () => {
     test.describe("new messages", () => {
         test.describe("in the main timeline", () => {
             test("Receiving a message makes a room unread", async ({

diff --git a/playwright/e2e/read-receipts/new-messages-thread-roots.spec.ts b/playwright/e2e/read-receipts/new-messages-thread-roots.spec.ts
@@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details.
 
 import { many, test } from ".";
 
-test.describe("Read receipts", () => {
+test.describe("Read receipts", { tag: "@mergequeue" }, () => {
     test.describe("new messages", () => {
         test.describe("thread roots", () => {
             test("Reading a thread root does not mark the thread as read", async ({

diff --git a/playwright/e2e/read-receipts/notifications.spec.ts b/playwright/e2e/read-receipts/notifications.spec.ts
@@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details.
 
 import { test } from ".";
 
-test.describe("Read receipts", () => {
+test.describe("Read receipts", { tag: "@mergequeue" }, () => {
     test.describe("Notifications", () => {
         test.describe("in the main timeline", () => {
             test.fixme("A new message that mentions me shows a notification", () => {});

diff --git a/playwright/e2e/read-receipts/reactions-in-threads.spec.ts b/playwright/e2e/read-receipts/reactions-in-threads.spec.ts
@@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details.
 
 import { test, expect } from ".";
 
-test.describe("Read receipts", () => {
+test.describe("Read receipts", { tag: "@mergequeue" }, () => {
     test.describe("reactions", () => {
         test.describe("in threads", () => {
             test("A reaction to a threaded message does not make the room unread", async ({

diff --git a/playwright/e2e/read-receipts/reactions-main-timeline.spec.ts b/playwright/e2e/read-receipts/reactions-main-timeline.spec.ts
@@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details.
 
 import { test } from ".";
 
-test.describe("Read receipts", () => {
+test.describe("Read receipts", { tag: "@mergequeue" }, () => {
     test.describe("reactions", () => {
         test.describe("in the main timeline", () => {
             test("Receiving a reaction to a message does not make a room unread", async ({

diff --git a/playwright/e2e/read-receipts/reactions-thread-roots.spec.ts b/playwright/e2e/read-receipts/reactions-thread-roots.spec.ts
@@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details.
 
 import { test } from ".";
 
-test.describe("Read receipts", () => {
+test.describe("Read receipts", { tag: "@mergequeue" }, () => {
     test.describe("reactions", () => {
         test.describe("thread roots", () => {
             test("A reaction to a thread root does not make the room unread", async ({

diff --git a/playwright/e2e/read-receipts/read-receipts.spec.ts b/playwright/e2e/read-receipts/read-receipts.spec.ts
@@ -13,7 +13,7 @@ import { ElementAppPage } from "../../pages/ElementAppPage";
 import { Bot } from "../../pages/bot";
 import { test } from ".";
 
-test.describe("Read receipts", () => {
+test.describe("Read receipts", { tag: "@mergequeue" }, () => {
     test.use({
         displayName: "Mae",
         botCreateOpts: { displayName: "Other User" },

diff --git a/playwright/e2e/read-receipts/redactions-in-threads.spec.ts b/playwright/e2e/read-receipts/redactions-in-threads.spec.ts
@@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details.
 
 import { test } from ".";
 
-test.describe("Read receipts", () => {
+test.describe("Read receipts", { tag: "@mergequeue" }, () => {
     test.describe("redactions", () => {
         test.describe("in threads", () => {
             test("Redacting the threaded message pointed to by my receipt leaves the room read", async ({

diff --git a/playwright/e2e/read-receipts/redactions-main-timeline.spec.ts b/playwright/e2e/read-receipts/redactions-main-timeline.spec.ts
@@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details.
 
 import { test } from ".";
 
-test.describe("Read receipts", () => {
+test.describe("Read receipts", { tag: "@mergequeue" }, () => {
     test.describe("redactions", () => {
         test.describe("in the main timeline", () => {
             test("Redacting the message pointed to by my receipt leaves the room read", async ({

diff --git a/playwright/e2e/read-receipts/redactions-thread-roots.spec.ts b/playwright/e2e/read-receipts/redactions-thread-roots.spec.ts
@@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details.
 
 import { test } from ".";
 
-test.describe("Read receipts", () => {
+test.describe("Read receipts", { tag: "@mergequeue" }, () => {
     test.describe("redactions", () => {
         test.describe("thread roots", () => {
             test("Redacting a thread root after it was read leaves the room read", async ({

diff --git a/playwright/e2e/read-receipts/room-list-order.spec.ts b/playwright/e2e/read-receipts/room-list-order.spec.ts
@@ -10,7 +10,7 @@ Please see LICENSE files in the repository root for full details.
 
 import { test } from ".";
 
-test.describe("Read receipts", () => {
+test.describe("Read receipts", { tag: "@mergequeue" }, () => {
     test.describe("Room list order", () => {
         test("Rooms with unread messages appear at the top of room list if 'unread first' is selected", async ({
             roomAlpha: room1,