test(browsers-smoketests): Run a set of smoke tests on both Chrome an…

…d Firefox

This commit introduces tape as the test framework used to define the tests in the
test extension contexts and send them to the nodejs script that orchestrate the
test run.

The nodejs script has also been migrated from mocha to tape, it uses the custom test
helpers provided to setup the test environment (e.g. create a temporary dir
for the test extension, copy the last polyfill build, bundle tape to be used
in the test extension, start the browser which run the test extension
and finally collect the results of the test extension) and then it merges all the
tap logs collected from every test extension into a single "per browser" test suite.

- updated travis nodejs environment to nodejs 8
- uses tape to collect test results from inside the test extension
- added test case to check polyfill 'existing browser API object' detection
- added test for expected rejection on tabs.sendMessage with an invalid tabId
- added test with multiple listeners which resolves to undefined and null
- optionally run chrome smoketests with --enable-features=NativeCrxBindings

.travis.yml

@@ -2,7 +2,8 @@ language: node_js
 sudo: false
 node_js:
 ## Some of the ES6 syntax used in the browser-polyfill sources is only supported on nodejs >= 6
-- '6'
+## and the selenium-webdriver dependency used by the integration tests requires nodejs >= 8.
+- '8'
 
 script:
 - npm run build
@@ -14,7 +15,14 @@ script:
 - export DISPLAY=:99.0
 - sh -e /etc/init.d/xvfb start
 - echo "RUN integration tests on chrome" &&
-  TRAVIS_CI=true ./test/run-chrome-smoketests.sh
+  TRAVIS_CI=true ./test/run-browsers-smoketests.sh
+
+## See https://docs.travis-ci.com/user/chrome
+sudo: required
+
+addons:
+  firefox: 'latest'
+  chrome: 'stable'
 
 after_script: npm run publish-coverage
 

package.json

@@ -22,9 +22,13 @@
     "babel-plugin-transform-es2015-modules-umd": "^6.24.1",
     "babel-preset-babili": "^0.0.10",
     "babel-preset-es2017": "^6.24.1",
+    "browserify": "^16.2.2",
     "chai": "^3.5.0",
+    "chromedriver": "^2.38.3",
     "eslint": "^3.9.1",
     "finalhandler": "^1.1.0",
+    "geckodriver": "^1.11.0",
+    "global-replaceify": "^1.0.0",
     "grunt": "^1.0.1",
     "grunt-babel": "^6.0.0",
     "grunt-contrib-concat": "^1.0.1",
@@ -35,9 +39,13 @@
     "jsdom": "^9.6.0",
     "mocha": "^3.1.0",
     "nyc": "^8.3.1",
-    "puppeteer": "^0.10.2",
+    "selenium-webdriver": "^4.0.0-alpha.1",
     "serve-static": "^1.13.1",
-    "sinon": "^1.17.6"
+    "shelljs": "^0.8.2",
+    "sinon": "^1.17.6",
+    "tap-nirvana": "^1.0.8",
+    "tape-async": "^2.3.0",
+    "tmp": "0.0.33"
   },
   "nyc": {
     "reporter": [
@@ -54,6 +62,6 @@
     "test": "mocha",
     "test-coverage": "COVERAGE=y nyc mocha",
     "test-minified": "TEST_MINIFIED_POLYFILL=1 mocha",
-    "test-integration": "mocha -r test/mocha-babel test/integration/test-*"
+    "test-integration": "tape test/integration/test-*"
   }
 }

test/fixtures/browserify-tape.js

@@ -0,0 +1,12 @@
+const browserify = require("browserify");
+
+const b = browserify();
+
+b.add("./test/fixtures/tape-standalone.js");
+b.transform("global-replaceify", {
+  global: true,
+  replacements: {
+    setImmediate: "require('timers').setImmediate",
+  },
+});
+b.bundle().pipe(process.stdout);

test/fixtures/detect-browser-api-object-in-content-script/content.js

@@ -0,0 +1,18 @@
+test("browser api object in content script", (t) => {
+  t.ok(browser && browser.runtime, "a global browser API object should be defined");
+  t.ok(chrome && chrome.runtime, "a global chrome API object should be defined");
+
+  if (navigator.userAgent.includes("Firefox/")) {
+    // Check that the polyfill didn't create a polyfill wrapped browser API object on Firefox.
+    t.equal(browser.runtime, chrome.runtime, "browser.runtime and chrome.runtime should be equal on Firefox");
+    // On Firefox, window is not the global object for content scripts, and so we expect window.browser to not
+    // be defined.
+    t.equal(window.browser, undefined, "window.browser is expected to be undefined on Firefox");
+  } else {
+    // Check that the polyfill has created a wrapped API namespace as expected.
+    t.notEqual(browser.runtime, chrome.runtime, "browser.runtime and chrome.runtime should not be equal");
+    // On chrome, window is the global object and so the polyfilled browser API should
+    // be also equal to window.browser.
+    t.equal(browser, window.browser, "browser and window.browser should be the same object");
+  }
+});

test/fixtures/detect-browser-api-object-in-content-script/manifest.json

@@ -0,0 +1,19 @@
+{
+  "manifest_version": 2,
+  "name": "test-detect-browser-api-object-in-content-script",
+  "version": "0.1",
+  "description": "test-detect-browser-api-object-in-content-script",
+  "content_scripts": [
+    {
+      "matches": [
+        "http://localhost/*"
+      ],
+      "js": [
+        "browser-polyfill.js",
+        "tape.js",
+        "content.js"
+      ]
+    }
+  ],
+  "permissions": []
+}

test/fixtures/multiple-onmessage-listeners-extension/background.js

@@ -0,0 +1,39 @@
+console.log(name, "background page loaded");
+
+async function testMessageHandler(msg, sender) {
+  console.log(name, "background received msg", {msg, sender});
+
+  // We only expect messages coming from a content script in this test.
+  if (!sender.tab || !msg.startsWith("test-multiple-onmessage-listeners:")) {
+    return {
+      success: false,
+      failureReason: `An unexpected message has been received: ${JSON.stringify({msg, sender})}`,
+    };
+  }
+
+  if (msg.endsWith(":resolve-to-undefined")) {
+    return undefined;
+  }
+
+  if (msg.endsWith(":resolve-to-null")) {
+    return null;
+  }
+
+  return {
+    success: false,
+    failureReason: `An unexpected message has been received: ${JSON.stringify({msg, sender})}`,
+  };
+}
+
+// Register the same message handler twice.
+browser.runtime.onMessage.addListener(testMessageHandler);
+browser.runtime.onMessage.addListener(testMessageHandler);
+
+// Register an additional message handler that always reply after
+// a small latency time.
+browser.runtime.onMessage.addListener(async (msg, sender) => {
+  await new Promise(resolve => setTimeout(resolve, 100));
+  return "resolved-to-string-with-latency";
+});
+
+console.log(name, "background page ready to receive a content script message...");

test/fixtures/multiple-onmessage-listeners-extension/content.js

@@ -0,0 +1,18 @@
+test("Multiple runtime.onmessage listeners which resolve to undefined", async (t) => {
+  const res = await browser.runtime.sendMessage("test-multiple-onmessage-listeners:resolve-to-undefined");
+
+  if (navigator.userAgent.includes("Firefox/")) {
+    t.deepEqual(res, undefined, "Got an undefined value as expected");
+  } else {
+    // NOTE: When an onMessage listener sends `undefined` in a response,
+    // Chrome internally converts it to null and the receiver receives it
+    // as a null object.
+    t.deepEqual(res, null, "Got a null value as expected on Chrome");
+  }
+});
+
+test("Multiple runtime.onmessage listeners which resolve to null", async (t) => {
+  const res = await browser.runtime.sendMessage("test-multiple-onmessage-listeners:resolve-to-null");
+
+  t.deepEqual(res, null, "Got a null value as expected");
+});

test/fixtures/multiple-onmessage-listeners-extension/manifest.json

@@ -0,0 +1,25 @@
+{
+  "manifest_version": 2,
+  "name": "test-multiple-onmessage-listeners",
+  "version": "0.1",
+  "description": "test-multiple-onmessage-listeners",
+  "content_scripts": [
+    {
+      "matches": [
+        "http://localhost/*"
+      ],
+      "js": [
+        "browser-polyfill.js",
+        "tape.js",
+        "content.js"
+      ]
+    }
+  ],
+  "permissions": [],
+  "background": {
+    "scripts": [
+      "browser-polyfill.js",
+      "background.js"
+    ]
+  }
+}

test/fixtures/runtime-messaging-extension/background.js

@@ -39,6 +39,12 @@ browser.runtime.onMessage.addListener((msg, sender, sendResponse) => {
     case "test - sendMessage with returned rejected Promise with non-Error value":
       return Promise.reject("rejected-non-error-value");
 
+    case "test - sendMessage with returned rejected Promise with non-Error value with message property":
+      return Promise.reject({message: "rejected-non-error-message"});
+
+    case "test - sendMessage with listener callback throws":
+      throw new Error("listener throws");
+
     default:
       return Promise.resolve(
         `Unxpected message received by the background page: ${JSON.stringify(msg)}\n`);

test/fixtures/runtime-messaging-extension/content.js

@@ -1,29 +1,26 @@
-const {name} = browser.runtime.getManifest();
-
-async function runTest() {
-  let reply;
-  reply = await browser.runtime.sendMessage("test - sendMessage with returned Promise reply");
-  console.log(name, "test - returned resolved Promise - received", reply);
-
-  reply = await browser.runtime.sendMessage("test - sendMessage with returned value reply");
-  console.log(name, "test - returned value - received", reply);
-
-  reply = await browser.runtime.sendMessage("test - sendMessage with synchronous sendResponse");
-  console.log(name, "test - synchronous sendResponse - received", reply);
-
-  reply = await browser.runtime.sendMessage("test - sendMessage with asynchronous sendResponse");
-  console.log(name, "test - asynchronous sendResponse - received", reply);
+test("sendMessage with returned Promise reply", async (t) => {
+  const reply = await browser.runtime.sendMessage("test - sendMessage with returned Promise reply");
+  t.equal(reply, "bg page reply 1");
+});
 
-  reply = await browser.runtime.sendMessage("test - second listener if the first does not reply");
-  console.log(name, "test - second listener sendResponse - received", reply);
+test("sendMessage with returned value reply", async (t) => {
+  const reply = await browser.runtime.sendMessage("test - sendMessage with returned value reply");
+  t.equal(reply, "second listener reply");
+});
 
-  console.log(name, "content script messages sent");
-}
+test("sendMessage with synchronous sendResponse", async (t) => {
+  const reply = await browser.runtime.sendMessage("test - sendMessage with synchronous sendResponse");
+  t.equal(reply, "bg page reply 3");
+});
 
-console.log(name, "content script loaded");
+test("sendMessage with asynchronous sendResponse", async (t) => {
+  const reply = await browser.runtime.sendMessage("test - sendMessage with asynchronous sendResponse");
+  t.equal(reply, "bg page reply 4");
+});
 
-runTest().catch((err) => {
-  console.error("content script error", err);
+test("second listener if the first does not reply", async (t) => {
+  const reply = await browser.runtime.sendMessage("test - second listener if the first does not reply");
+  t.equal(reply, "second listener reply");
 });
 
 test("sendMessage with returned rejected Promise with Error value", async (t) => {
@@ -33,6 +30,7 @@ test("sendMessage with returned rejected Promise with Error value", async (t) =>
     t.fail(`Unexpected successfully reply while expecting a rejected promise`);
     t.equal(reply, undefined, "Unexpected successfully reply");
   } catch (err) {
+    t.ok(err instanceof Error, "Got an error object as expected");
     t.equal(err.message, "rejected-error-value", "Got an error rejection with the expected message");
   }
 });
@@ -44,9 +42,36 @@ test("sendMessage with returned rejected Promise with non-Error value", async (t
     t.fail(`Unexpected successfully reply while expecting a rejected promise`);
     t.equal(reply, undefined, "Unexpected successfully reply");
   } catch (err) {
-    // Unfortunately Firefox currently reject an error with an undefined
-    // message, in the meantime we just check that the object rejected is
-    // an instance of Error.
+    // Unfortunately Firefox currently rejects an error with an "undefined"
+    // message in Firefox 60 and "An unexpected error occurred" in Firefox 59,
+    // in the meantime we just check that the object rejected is an instance
+    // of Error.
+    t.ok(err instanceof Error, "Got an error object as expected");
+  }
+});
+
+test("sendMessage with returned rejected Promise with non-Error value with message property", async (t) => {
+  try {
+    const reply = await browser.runtime.sendMessage(
+      "test - sendMessage with returned rejected Promise with non-Error value with message property");
+    t.fail(`Unexpected successfully reply while expecting a rejected promise`);
+    t.equal(reply, undefined, "Unexpected successfully reply");
+  } catch (err) {
+    // Firefox currently converts any rejection with a message property into an error instance
+    // with the value of that message property as the error message.
+    t.ok(err instanceof Error, "Got an error object as expected");
+    t.equal(err.message, "rejected-non-error-message", "Got an error rejection with the expected message");
+  }
+});
+
+test("sendMessage with listener callback throws", async (t) => {
+  try {
+    const reply = await browser.runtime.sendMessage(
+      "test - sendMessage with listener callback throws");
+    t.fail(`Unexpected successfully reply while expecting a rejected promise`);
+    t.equal(reply, undefined, "Unexpected successfully reply");
+  } catch (err) {
     t.ok(err instanceof Error, "Got an error object as expected");
+    t.equal(err.message, "listener throws", "Got an error with the expected message");
   }
 });

test/fixtures/runtime-messaging-extension/manifest.json

@@ -10,6 +10,7 @@
       ],
       "js": [
         "browser-polyfill.js",
+        "tape.js",
         "content.js"
       ]
     }

test/fixtures/tabs-sendmessage-extension/background.js

@@ -0,0 +1,36 @@
+console.log(name, "background page loaded");
+
+browser.runtime.onMessage.addListener(async (msg, sender, sendResponse) => {
+  console.log(name, "background received msg", {msg, sender});
+
+  // We only expect messages coming from a content script in this test.
+  if (!sender.tab || msg != "test-tabssendMessage-unknown-tabid") {
+    return {
+      success: false,
+      failureReason: `An unexpected message has been received: ${JSON.stringify({msg, sender})}`,
+    };
+  }
+
+  try {
+    const tabs = await browser.tabs.query({});
+    const lastValidTabId = tabs.reduce((acc, tab) => {
+      return Math.max(acc, tab.id);
+    }, 0);
+    const INVALID_TABID = lastValidTabId + 100;
+
+    await browser.tabs.sendMessage(INVALID_TABID, "message-to-unknown-tab");
+
+    return {
+      success: false,
+      failureReason: `browser.tabs.sendMessage should reject on sending messages to non-existing tab`,
+    };
+  } catch (err) {
+    return {
+      success: true,
+      isRejected: true,
+      errorMessage: err.message,
+    };
+  }
+});
+
+console.log(name, "background page ready to receive a content script message...");

test/fixtures/tabs-sendmessage-extension/content.js

@@ -0,0 +1,8 @@
+test("tabs.sendMessage reject when sending to unknown tab id", async (t) => {
+  const res = await browser.runtime.sendMessage("test-tabssendMessage-unknown-tabid");
+  t.deepEqual(res, {
+    success: true,
+    isRejected: true,
+    errorMessage: "Could not establish connection. Receiving end does not exist.",
+  }, "The background page got a rejection as expected");
+});

test/fixtures/tabs-sendmessage-extension/manifest.json

@@ -0,0 +1,25 @@
+{
+  "manifest_version": 2,
+  "name": "test-tabs-sendmessage",
+  "version": "0.1",
+  "description": "test-tabs-sendmessage",
+  "content_scripts": [
+    {
+      "matches": [
+        "http://localhost/*"
+      ],
+      "js": [
+        "browser-polyfill.js",
+        "tape.js",
+        "content.js"
+      ]
+    }
+  ],
+  "permissions": [],
+  "background": {
+    "scripts": [
+      "browser-polyfill.js",
+      "background.js"
+    ]
+  }
+}