[wptrunner] Support testdriver.js in {ref,print-ref,crash}tests (#48486)

* [wptrunner] Add `testdriver` flag to non-testharness manifest items This will let wptrunner gracefully skip testdriver tests if the executor doesn't support them. * [wptrunner] Support testdriver.js in {ref,print-ref,crash}tests Non-testharness executors that don't support testdriver simply skip those tests.
web-platform-tests · Dec 12, 2024 · c715b79 · c715b79
1 parent 5c0247d
commit c715b79
Show file tree

Hide file tree

Showing 18 changed files with 284 additions and 61 deletions.
diff --git a/docs/writing-tests/testdriver.md b/docs/writing-tests/testdriver.md
@@ -13,8 +13,12 @@ written purely using web platform APIs. Outside of automation
 contexts, it allows human operators to provide expected input
 manually (for operations which may be described in simple terms).
 
-It is currently supported only for [testharness.js](testharness)
-tests.
+testdriver.js supports the following test types:
+* [testharness.js](testharness) tests
+* [reftests](reftests) and [print-reftests](print-reftests) that use the
+  `class=reftest-wait` attribute on the root element to control completion
+* [crashtests](crashtest) that use the `class=test-wait` attribute to control
+  completion
 
 ## Markup ##
 

diff --git a/infrastructure/crashtests/testdriver.html b/infrastructure/crashtests/testdriver.html
@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html class="test-wait">
+<title>crashtests support testdriver.js</title>
+<script src="/resources/testdriver.js"></script>
+<script src="/resources/testdriver-vendor.js"></script>
+<button>Complete the test</button>
+<script>
+  const button = document.querySelector("button");
+  button.addEventListener("click", () => {
+    document.documentElement.classList.remove("test-wait");
+  });
+  test_driver.click(button);
+</script>
diff --git a/infrastructure/metadata/infrastructure/reftest/testdriver-in-ref.html.ini b/infrastructure/metadata/infrastructure/reftest/testdriver-in-ref.html.ini
@@ -0,0 +1,5 @@
+[testdriver-in-ref.html]
+  disabled:
+    # https://github.com/web-platform-tests/wpt/issues/13183
+    if product == "firefox" or product == "firefox_android":
+      "marionette executor doesn't currently implement testdriver for reftests"
diff --git a/infrastructure/reftest/testdriver-child.html b/infrastructure/reftest/testdriver-child.html
@@ -0,0 +1,19 @@
+<!DOCTYPE html>
+<script src="/resources/testdriver.js"></script>
+<script src="/resources/testdriver-vendor.js"></script>
+<style>
+  :root {
+    background-color: red;
+  }
+</style>
+<button>Turn green</button>
+<script>
+  test_driver.set_test_context(parent);
+  const button = document.querySelector("button");
+  button.addEventListener("click", () => {
+    button.remove();
+    document.documentElement.style.backgroundColor = "green";
+    test_driver.message_test("done");
+  });
+  test_driver.click(button);
+</script>
diff --git a/infrastructure/reftest/testdriver-iframe.sub.html b/infrastructure/reftest/testdriver-iframe.sub.html
@@ -0,0 +1,29 @@
+<!DOCTYPE html>
+<html class="reftest-wait">
+<title>reftests support testdriver.js in iframes</title>
+<link rel="match" href="green.html">
+<script src="/resources/testdriver.js"></script>
+<script src="/resources/testdriver-vendor.js"></script>
+<style>
+  body {
+    margin: 0;
+    padding: 0;
+    width: 100%;
+    height: 100%;
+  }
+  iframe {
+    position: absolute;
+    border: none;
+    width: inherit;
+    height: inherit;
+  }
+</style>
+<script>
+  // Attach the handler that completes the test before loading the calling iframe.
+  window.addEventListener("message", (evt) => {
+    if (evt.data === "done") {
+      document.documentElement.classList.remove("reftest-wait");
+    }
+  });
+</script>
+<iframe src="https://{{host}}:{{ports[https][1]}}/infrastructure/reftest/testdriver-child.html"></iframe>
diff --git a/infrastructure/reftest/testdriver-in-ref.html b/infrastructure/reftest/testdriver-in-ref.html
@@ -0,0 +1,6 @@
+<!DOCTYPE html>
+<title>references support testdriver.js</title>
+<link rel="match" href="testdriver.html">
+<style>
+  :root {background-color:green}
+</style>
diff --git a/infrastructure/reftest/testdriver-print.html b/infrastructure/reftest/testdriver-print.html
@@ -0,0 +1,23 @@
+<!DOCTYPE html>
+<html class="reftest-wait">
+<title>print-reftests support testdriver.js</title>
+<link rel="match" href="reftest_match-print-ref.html">
+<script src="/resources/testdriver.js"></script>
+<script src="/resources/testdriver-vendor.js"></script>
+<style>
+  * { margin: 0; padding: 0; }
+  div { page-break-after: always; }
+</style>
+<button>Add a page</button>
+<div>page 1</div>
+<script>
+  const button = document.querySelector("button");
+  button.addEventListener("click", () => {
+    button.remove();
+    const page2 = document.createElement("div");
+    page2.innerText = "page 2";
+    document.body.appendChild(page2);
+    document.documentElement.classList.remove("reftest-wait");
+  });
+  test_driver.click(button);
+</script>
diff --git a/infrastructure/reftest/testdriver.html b/infrastructure/reftest/testdriver.html
@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html class="reftest-wait">
+<title>reftests support testdriver.js</title>
+<link rel="match" href="green.html">
+<script src="/resources/testdriver.js"></script>
+<script src="/resources/testdriver-vendor.js"></script>
+<style>
+  :root {
+    background-color: red;
+  }
+</style>
+<button>Turn green</button>
+<script>
+  const button = document.querySelector("button");
+  button.addEventListener("click", () => {
+    button.remove();
+    document.documentElement.style.backgroundColor = "green";
+    document.documentElement.classList.remove("reftest-wait");
+  });
+  test_driver.click(button);
+</script>
diff --git a/lint.ignore b/lint.ignore
@@ -769,6 +769,12 @@ HTML INVALID SYNTAX: quirks/percentage-height-calculation.html
 HTML INVALID SYNTAX: trusted-types/TrustedTypePolicyFactory-getAttributeType-namespace.html
 
 # Tests which include testdriver.js but aren't testharness.js tests
+# TODO(web-platform-tests/wpt#13183): Remove this rule once support is added.
+TESTDRIVER-IN-UNSUPPORTED-TYPE: infrastructure/crashtests/testdriver.html
+TESTDRIVER-IN-UNSUPPORTED-TYPE: infrastructure/reftest/testdriver.html
+TESTDRIVER-IN-UNSUPPORTED-TYPE: infrastructure/reftest/testdriver-child.html
+TESTDRIVER-IN-UNSUPPORTED-TYPE: infrastructure/reftest/testdriver-iframe.sub.html
+TESTDRIVER-IN-UNSUPPORTED-TYPE: infrastructure/reftest/testdriver-print.html
 TESTDRIVER-IN-UNSUPPORTED-TYPE: css/css-grid/grid-model/grid-layout-stale-001.html
 TESTDRIVER-IN-UNSUPPORTED-TYPE: css/css-grid/grid-model/grid-layout-stale-002.html
 TESTDRIVER-IN-UNSUPPORTED-TYPE: css/css-scroll-anchoring/fullscreen-crash.html

diff --git a/tools/manifest/item.py b/tools/manifest/item.py
@@ -166,7 +166,7 @@ def pac(self) -> Optional[Text]:
         return self._extras.get("pac")
 
     @property
-    def testdriver(self) -> Optional[Text]:
+    def testdriver(self) -> Optional[bool]:
         return self._extras.get("testdriver")
 
     @property
@@ -240,6 +240,10 @@ def fuzzy(self) -> Fuzzy:
             rv[key] = v
         return rv
 
+    @property
+    def testdriver(self) -> Optional[bool]:
+        return self._extras.get("testdriver")
+
     def to_json(self) -> Tuple[Optional[Text], List[Tuple[Text, Text]], Dict[Text, Any]]:  # type: ignore
         rel_url = None if self._url == self.path else self._url
         rv: Tuple[Optional[Text], List[Tuple[Text, Text]], Dict[Text, Any]] = (rel_url, self.references, {})
@@ -252,6 +256,8 @@ def to_json(self) -> Tuple[Optional[Text], List[Tuple[Text, Text]], Dict[Text, A
             extras["dpi"] = self.dpi
         if self.fuzzy:
             extras["fuzzy"] = list(self.fuzzy.items())
+        if self.testdriver:
+            extras["testdriver"] = self.testdriver
         return rv
 
     @classmethod
@@ -315,6 +321,16 @@ class CrashTest(URLManifestItem):
     def timeout(self) -> Optional[Text]:
         return None
 
+    @property
+    def testdriver(self) -> Optional[bool]:
+        return self._extras.get("testdriver")
+
+    def to_json(self):  # type: ignore
+        rel_url, extras = super().to_json()
+        if self.testdriver:
+            extras["testdriver"] = self.testdriver
+        return rel_url, extras
+
 
 class WebDriverSpecTest(URLManifestItem):
     __slots__ = ()

diff --git a/tools/manifest/sourcefile.py b/tools/manifest/sourcefile.py
@@ -946,7 +946,8 @@ def manifest_items(self) -> Tuple[Text, List[ManifestItem]]:
                     self.tests_root,
                     self.rel_path,
                     self.url_base,
-                    self.rel_url
+                    self.rel_url,
+                    testdriver=self.has_testdriver,
                 )]
 
         elif self.name_is_print_reftest:
@@ -965,6 +966,7 @@ def manifest_items(self) -> Tuple[Text, List[ManifestItem]]:
                     viewport_size=self.viewport_size,
                     fuzzy=self.fuzzy,
                     page_ranges=self.page_ranges,
+                    testdriver=self.has_testdriver,
                 )]
 
         elif self.name_is_multi_global:
@@ -1065,7 +1067,8 @@ def manifest_items(self) -> Tuple[Text, List[ManifestItem]]:
                     timeout=self.timeout,
                     viewport_size=self.viewport_size,
                     dpi=self.dpi,
-                    fuzzy=self.fuzzy
+                    fuzzy=self.fuzzy,
+                    testdriver=self.has_testdriver,
                 ))
 
         elif self.content_is_css_visual and not self.name_is_reference:

diff --git a/tools/manifest/tests/test_manifest.py b/tools/manifest/tests/test_manifest.py
@@ -335,3 +335,29 @@ def test_manifest_spec_to_json():
             ]}},
         }
     }
+
+
+@pytest.mark.parametrize("testdriver,expected_extra", [
+    (True, {"testdriver": True}),
+    # Don't bloat the manifest with the `testdriver=False` default.
+    (False, {}),
+])
+def test_dump_testdriver(testdriver, expected_extra):
+    m = manifest.Manifest("")
+    source_file = SourceFileWithTest("a" + os.path.sep + "b", "0"*40, item.RefTest,
+                                     testdriver=testdriver)
+
+    tree, sourcefile_mock = tree_and_sourcefile_mocks([(source_file, None, True)])
+    with mock.patch("tools.manifest.manifest.SourceFile", side_effect=sourcefile_mock):
+        assert m.update(tree) is True
+
+    assert m.to_json() == {
+        'version': 9,
+        'url_base': '/',
+        'items': {
+            'reftest': {'a': {'b': [
+                '0000000000000000000000000000000000000000',
+                (mock.ANY, [], expected_extra)
+            ]}},
+        }
+    }
diff --git a/tools/wptrunner/wptrunner/executors/executorwebdriver.py b/tools/wptrunner/wptrunner/executors/executorwebdriver.py
@@ -1031,8 +1031,9 @@ def get_or_create_test_window(self, protocol):
         return test_window
 
 
-class WebDriverRefTestExecutor(RefTestExecutor):
+class WebDriverRefTestExecutor(RefTestExecutor, TestDriverExecutorMixin):
     protocol_cls = WebDriverProtocol
+    supports_testdriver = True
 
     def __init__(self, logger, browser, server_config, timeout_multiplier=1,
                  screenshot_cache=None, close_after_done=True,
@@ -1056,7 +1057,8 @@ def __init__(self, logger, browser, server_config, timeout_multiplier=1,
         self.debug_test = debug_test
 
         with open(os.path.join(here, "test-wait.js")) as f:
-            self.wait_script = f.read() % {"classname": "reftest-wait"}
+            wait_script = f.read() % {"classname": "reftest-wait"}
+        TestDriverExecutorMixin.__init__(self, wait_script)
 
     def reset(self):
         self.implementation.reset()
@@ -1102,8 +1104,9 @@ def screenshot(self, test, viewport_size, dpi, page_ranges):
                             self.extra_timeout).run()
 
     def _screenshot(self, protocol, url, timeout):
-        self.protocol.base.load(url)
-        self.protocol.base.execute_script(self.wait_script, True)
+        # There's nothing we want from the "complete" message, so discard the
+        # return value.
+        self.run_testdriver(protocol, url, timeout)
 
         screenshot = self.protocol.webdriver.screenshot()
         if screenshot is None:
@@ -1148,8 +1151,9 @@ def screenshot(self, test, viewport_size, dpi, page_ranges):
                             self.extra_timeout).run()
 
     def _render(self, protocol, url, timeout):
-        protocol.webdriver.url = url
-        protocol.base.execute_script(self.wait_script, asynchronous=True)
+        # There's nothing we want from the "complete" message, so discard the
+        # return value.
+        self.run_testdriver(protocol, url, timeout)
 
         pdf = protocol.pdf_print.render_as_pdf(*self.viewport_size)
         screenshots = protocol.pdf_print.pdf_to_png(pdf, self.page_ranges)
@@ -1161,8 +1165,9 @@ def _render(self, protocol, url, timeout):
         return screenshots
 
 
-class WebDriverCrashtestExecutor(CrashtestExecutor):
+class WebDriverCrashtestExecutor(CrashtestExecutor, TestDriverExecutorMixin):
     protocol_cls = WebDriverProtocol
+    supports_testdriver = True
 
     def __init__(self, logger, browser, server_config, timeout_multiplier=1,
                  screenshot_cache=None, close_after_done=True,
@@ -1180,7 +1185,8 @@ def __init__(self, logger, browser, server_config, timeout_multiplier=1,
                                           capabilities=capabilities)
 
         with open(os.path.join(here, "test-wait.js")) as f:
-            self.wait_script = f.read() % {"classname": "test-wait"}
+            wait_script = f.read() % {"classname": "test-wait"}
+        TestDriverExecutorMixin.__init__(self, wait_script)
 
     def do_test(self, test):
         timeout = (test.timeout * self.timeout_multiplier if self.debug_info is None
@@ -1199,8 +1205,9 @@ def do_test(self, test):
         return (test.make_result(*data), [])
 
     def do_crashtest(self, protocol, url, timeout):
-        protocol.base.load(url)
-        protocol.base.execute_script(self.wait_script, asynchronous=True)
+        # There's nothing we want from the "complete" message, so discard the
+        # return value.
+        self.run_testdriver(protocol, url, timeout)
         result = {"status": "PASS", "message": None}
         if (leak_part := getattr(protocol, "leak", None)) and (counters := leak_part.check()):
             result["extra"] = {"leak_counters": counters}

diff --git a/tools/wptrunner/wptrunner/executors/message-queue.js b/tools/wptrunner/wptrunner/executors/message-queue.js
@@ -54,15 +54,19 @@
     case "complete":
       var tests = data.tests;
       var status = data.status;
-
-      var subtest_results = tests.map(function(x) {
-        return [x.name, x.status, x.message, x.stack];
-      });
-      payload = [status.status,
-                 status.message,
-                 status.stack,
-                 subtest_results];
-      clearTimeout(window.__wptrunner_timer);
+      if (tests && status) {
+        var subtest_results = tests.map(function(x) {
+          return [x.name, x.status, x.message, x.stack];
+        });
+        payload = [status.status,
+                   status.message,
+                   status.stack,
+                   subtest_results];
+        clearTimeout(window.__wptrunner_timer);
+      } else {
+        // Non-testharness test.
+        payload = [];
+      }
       break;
     case "action":
       payload = data;