Bug 1598763 [wpt PR 20409] - wptrunner: Fail with NotImplementedError when appropriate, a=testonly

Automatic update from web-platform-tests
wptrunner: Fail with NotImplementedError when appropriate (#20409)

Modify the base executor so that if a certain protocol message is not
implemented, we fail with NotImplementedError instead of raising an
exception and crashing the browser. Have unimplemented APIs
explicitly throw this error on marionette.

--

wpt-commits: 54bed30a94eb09200f8fa3d0dbf7360c271db2df
wpt-pr: 20409

Author: nsatragno

testing/web-platform/tests/infrastructure/metadata/infrastructure/testdriver/generate_test_report.html.ini

@@ -1,4 +1,4 @@
 [generate_test_report.html]
-  expected:
-    if product == "firefox": ERROR
-    if product == "safari" or product == "epiphany" or product == "webkit": ERROR
+  [TestDriver generate_test_report method]
+    expected:
+      if product == "firefox" or product == "safari" or product == "epiphany" or product == "webkit": FAIL

testing/web-platform/tests/infrastructure/metadata/infrastructure/testdriver/set_permission.https.html.ini

@@ -1,3 +1,8 @@
 [set_permission.https.html]
-  expected:
-    if product != "chrome": ERROR
+  [Grant Permission for one realm]
+    expected:
+      if product != "chrome": FAIL
+
+  [Deny Permission, omit one realm]
+    expected:
+      if product != "chrome": FAIL

testing/web-platform/tests/infrastructure/metadata/infrastructure/testdriver/virtual_authenticator.html.ini

@@ -1,3 +1,28 @@
 [virtual_authenticator.html]
-  expected:
-    if product == "firefox" or product == "safari" or product == "epiphany" or product == "webkit": ERROR
+  [Can create an authenticator]
+    expected:
+      if product == "firefox" or product == "safari" or product == "epiphany" or product == "webkit": FAIL
+
+  [Can add a credential]
+    expected:
+      if product == "firefox" or product == "safari" or product == "epiphany" or product == "webkit": FAIL
+
+  [Can get the credentials]
+    expected:
+      if product == "firefox" or product == "safari" or product == "epiphany" or product == "webkit": FAIL
+
+  [Can remove a credential]
+    expected:
+      if product == "firefox" or product == "safari" or product == "epiphany" or product == "webkit": FAIL
+
+  [Can remove all credentials]
+    expected:
+      if product == "firefox" or product == "safari" or product == "epiphany" or product == "webkit": FAIL
+
+  [Can set user verified]
+    expected:
+      if product == "firefox" or product == "safari" or product == "epiphany" or product == "webkit": FAIL
+
+  [Can remove a virtual authenticator]
+    expected:
+      if product == "firefox" or product == "safari" or product == "epiphany" or product == "webkit": FAIL

testing/web-platform/tests/infrastructure/testdriver/virtual_authenticator.html

@@ -24,7 +24,7 @@
   isResidentCredential: false,
 };
 
-let authenticator_id;
+let authenticator_id = null;
 
 promise_test(async t => {
   authenticator_id = await test_driver.add_virtual_authenticator({

testing/web-platform/tests/tools/wptrunner/wptrunner/executors/base.py

@@ -13,6 +13,7 @@
 
 from ..testrunner import Stop
 from .protocol import Protocol, BaseProtocolPart
+import webdriver as client
 
 here = os.path.split(__file__)[0]
 
@@ -700,6 +701,9 @@ def process_action(self, url, payload):
             raise ValueError("Unknown action %s" % action)
         try:
             result = action_handler(payload)
+        except (NotImplementedError, client.UnknownCommandException):
+            self.logger.warning("Action %s not implemented" % action)
+            self._send_message("complete", "error", "Action %s not implemented" % action)
         except Exception:
             self.logger.warning("Action %s failed" % action)
             self.logger.warning(traceback.format_exc())

testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py

@@ -31,7 +31,10 @@
                        ClickProtocolPart,
                        SendKeysProtocolPart,
                        TestDriverProtocolPart,
-                       CoverageProtocolPart)
+                       CoverageProtocolPart,
+                       GenerateTestReportProtocolPart,
+                       VirtualAuthenticatorProtocolPart,
+                       SetPermissionProtocolPart)
 from ..testrunner import Stop
 from ..webdriver_server import GeckoDriverServer
 
@@ -481,6 +484,44 @@ def dump(self):
                 # This usually happens if the process crashed
                 pass
 
+class MarionetteGenerateTestReportProtocolPart(GenerateTestReportProtocolPart):
+    def setup(self):
+        self.marionette = self.parent.marionette
+
+    def generate_test_report(self, config):
+        raise NotImplementedError("generate_test_report not yet implemented")
+
+class MarionetteVirtualAuthenticatorProtocolPart(VirtualAuthenticatorProtocolPart):
+    def setup(self):
+        self.marionette = self.parent.marionette
+
+    def add_virtual_authenticator(self, config):
+        raise NotImplementedError("add_virtual_authenticator not yet implemented")
+
+    def remove_virtual_authenticator(self, authenticator_id):
+        raise NotImplementedError("remove_virtual_authenticator not yet implemented")
+
+    def add_credential(self, authenticator_id, credential):
+        raise NotImplementedError("add_credential not yet implemented")
+
+    def get_credentials(self, authenticator_id):
+        raise NotImplementedError("get_credentials not yet implemented")
+
+    def remove_credential(self, authenticator_id, credential_id):
+        raise NotImplementedError("remove_credential not yet implemented")
+
+    def remove_all_credentials(self, authenticator_id):
+        raise NotImplementedError("remove_all_credentials not yet implemented")
+
+    def set_user_verified(self, authenticator_id, uv):
+        raise NotImplementedError("set_user_verified not yet implemented")
+
+class MarionetteSetPermissionProtocolPart(SetPermissionProtocolPart):
+    def setup(self):
+        self.marionette = self.parent.marionette
+
+    def set_permission(self, name, state, one_realm):
+        raise NotImplementedError("set_permission not yet implemented")
 
 class MarionetteProtocol(Protocol):
     implements = [MarionetteBaseProtocolPart,
@@ -493,7 +534,10 @@ class MarionetteProtocol(Protocol):
                   MarionetteActionSequenceProtocolPart,
                   MarionetteTestDriverProtocolPart,
                   MarionetteAssertsProtocolPart,
-                  MarionetteCoverageProtocolPart]
+                  MarionetteCoverageProtocolPart,
+                  MarionetteGenerateTestReportProtocolPart,
+                  MarionetteVirtualAuthenticatorProtocolPart,
+                  MarionetteSetPermissionProtocolPart]
 
     def __init__(self, executor, browser, capabilities=None, timeout_multiplier=1, e10s=True, ccov=False):
         do_delayed_imports()

testing/web-platform/tests/tools/wptrunner/wptrunner/testdriver-extra.js

@@ -19,7 +19,7 @@
             result = JSON.parse(data.message).result
             pending_resolve(result);
         } else {
-            pending_reject();
+            pending_reject(`${data.status}: ${data.message}`);
         }
     });
 

testing/web-platform/tests/tools/wptrunner/wptrunner/tests/__init__.py

@@ -0,0 +1,7 @@
+import os
+import sys
+
+here = os.path.abspath(os.path.split(__file__)[0])
+sys.path.insert(0, os.path.join(here, os.pardir, os.pardir, os.pardir))
+
+import localpaths as _localpaths  # noqa: F401