ci: use @playwright/test

apps/web/.gitignore

@@ -28,4 +28,5 @@ __diff_output__
 
 dist
 public/workbox
-scripts/secrets
+scripts/secrets
+test-results

apps/web/__e2e__/navigation.test.js

@@ -1,17 +1,16 @@
 /* eslint-disable no-undef */
 
+const { test, expect } = require("@playwright/test");
 const { getTestId } = require("./utils");
-const { toMatchImageSnapshot } = require("jest-image-snapshot");
 
-expect.extend({ toMatchImageSnapshot });
-
-beforeEach(async () => {
-  await page.goto("http://localhost:3000/");
+test.beforeEach(async ({ page }) => {
+  await page.goto("http://localhost:3000");
 });
 
 function createRoute(key, header) {
   return { buttonId: `navitem-${key}`, header };
 }
+
 const routes = [
   createRoute("notes", "Notes"),
   createRoute("notebooks", "Notebooks"),
@@ -19,20 +18,21 @@ const routes = [
   createRoute("tags", "Tags"),
   createRoute("trash", "Trash"),
   createRoute("settings", "Settings"),
-].map((route) => [route.header, route]);
+];
 
-test.each(routes)("navigating to %s", async (_header, route) => {
-  await page.waitForSelector(getTestId(route.buttonId), {
-    state: "visible",
+for (let route of routes) {
+  test(`navigating to ${route.header}`, async ({ page }) => {
+    await page.waitForSelector(getTestId(route.buttonId), {
+      state: "visible",
+    });
+    await page.click(getTestId(route.buttonId));
+    await expect(page.textContent(getTestId("routeHeader"))).resolves.toBe(
+      route.header
+    );
+    await page.waitForTimeout(300);
+    const navItem = await page.$(getTestId(route.buttonId));
+    await expect(navItem.screenshot()).resolves.toMatchSnapshot(
+      `nav-item-${route.buttonId}.png`
+    );
   });
-  await page.click(getTestId(route.buttonId));
-  await expect(page.textContent(getTestId("routeHeader"))).resolves.toBe(
-    route.header
-  );
-  const navItem = await page.$(getTestId(route.buttonId));
-  await expect(navItem.screenshot()).resolves.toMatchImageSnapshot({
-    failureThreshold: 5,
-    failureThresholdType: "percent",
-    allowSizeMismatch: true,
-  });
-});
+}

apps/web/__e2e__/notebooks.test.js

@@ -1,7 +1,6 @@
 /* eslint-disable no-undef */
+const { test, expect } = require("@playwright/test");
 const { getTestId, createNote, NOTE, NOTEBOOK } = require("./utils");
-const { toMatchImageSnapshot } = require("jest-image-snapshot");
-expect.extend({ toMatchImageSnapshot });
 const {
   navigateTo,
   openContextMenu,
@@ -13,15 +12,14 @@ const List = require("./utils/listitemidbuilder");
 const Menu = require("./utils/menuitemidbuilder");
 const { checkNotePresence, isPresent } = require("./utils/conditions");
 
-jest.setTimeout(35 * 1000);
+/**
+ * @type {Page}
+ */
+global.page = null;
 
-beforeEach(async () => {
+test.beforeEach(async ({ page: _page }) => {
+  global.page = _page;
   await page.goto("http://localhost:3000/");
-}, 600000);
-
-afterEach(async () => {
-  page.close();
-  page = await browser.newPage();
 });
 
 async function fillNotebookDialog(notebook) {
@@ -113,7 +111,7 @@ async function deleteNotebookAndCheckAbsence(notebookSelector) {
   await navigateTo("notebooks");
 }
 
-test("create a notebook", createNotebookAndCheckPresence);
+test("create a notebook", async () => await createNotebookAndCheckPresence());
 
 test("create a note inside a notebook", async () => {
   const notebookSelector = await createNotebookAndCheckPresence();
@@ -220,11 +218,4 @@ test("pin a notebook", async () => {
 
   // wait for the menu to properly close
   await page.waitForTimeout(500);
-
-  const notebook = await page.$(List.new("notebook").atIndex(0).build());
-  await expect(notebook.screenshot()).resolves.toMatchImageSnapshot({
-    failureThreshold: 5,
-    failureThresholdType: "percent",
-    allowSizeMismatch: true,
-  });
 });

apps/web/__e2e__/notes.test.js

@@ -5,9 +5,8 @@
  * TODO: We are still not checking if toast appears on delete/restore or not.
  */
 
+const { test, expect } = require("@playwright/test");
 const { getTestId, createNote, NOTE } = require("./utils");
-const { toMatchImageSnapshot } = require("jest-image-snapshot");
-expect.extend({ toMatchImageSnapshot });
 const {
   navigateTo,
   clickMenuItem,
@@ -25,13 +24,7 @@ const {
 const List = require("./utils/listitemidbuilder");
 const Menu = require("./utils/menuitemidbuilder");
 
-jest.setTimeout(35 * 1000);
-
-// const testCISkip = process.env.CI ? test.skip : test;
-
-var createNoteAndCheckPresence = async function createNoteAndCheckPresence(
-  note = NOTE
-) {
+async function createNoteAndCheckPresence(note = NOTE) {
   await createNote(note, "notes");
 
   // make sure the note has saved.
@@ -42,9 +35,7 @@ var createNoteAndCheckPresence = async function createNoteAndCheckPresence(
   await page.click(noteSelector, { button: "left" });
 
   return noteSelector;
-};
-
-const staticCreateNoteAndCheckPresence = createNoteAndCheckPresence.bind(this);
+}
 
 async function deleteNoteAndCheckAbsence() {
   const noteSelector = await createNoteAndCheckPresence();
@@ -93,11 +84,7 @@ async function checkNotePinned(noteSelector, pause) {
   await page.waitForTimeout(500);
 
   const note = await page.$(List.new("note").grouped().atIndex(0).build());
-  await expect(note.screenshot()).resolves.toMatchImageSnapshot({
-    failureThreshold: 5,
-    allowSizeMismatch: true,
-    failureThresholdType: "percent",
-  });
+  await expect(note.screenshot()).resolves.toMatchSnapshot("note-pinned.png");
 }
 
 async function checkNoteLocked(noteSelector) {
@@ -129,11 +116,7 @@ async function checkNoteColored(noteSelector) {
   await page.waitForTimeout(500);
 
   const note = await page.$(List.new("note").grouped().atIndex(0).build());
-  await expect(note.screenshot()).resolves.toMatchImageSnapshot({
-    failureThreshold: 5,
-    allowSizeMismatch: true,
-    failureThresholdType: "percent",
-  });
+  await expect(note.screenshot()).resolves.toMatchSnapshot("note-colored.png");
 
   await navigateTo("red");
 
@@ -142,11 +125,9 @@ async function checkNoteColored(noteSelector) {
 
   const coloredNote = await page.$(List.new("note").atIndex(0).build());
   if (!coloredNote) throw new Error("Colored note not present.");
-  await expect(coloredNote.screenshot()).resolves.toMatchImageSnapshot({
-    allowSizeMismatch: true,
-    failureThreshold: 5,
-    failureThresholdType: "percent",
-  });
+  await expect(coloredNote.screenshot()).resolves.toMatchSnapshot(
+    "note-colored-2.png"
+  );
 }
 
 async function addNoteToNotebook() {
@@ -169,59 +150,26 @@ async function addNoteToNotebook() {
   await checkNotePresence(0, false);
 }
 
-describe.each(["independent", "sequential"])("run tests %sly", (type) => {
-  beforeAll(async () => {
-    if (type === "sequential") {
-      await page.goto("http://localhost:3000/");
-    }
-  });
-  // clear all browser data after running all tests for a single case
-  // so this will clear all data after running test independently & sequentially.
-  afterAll(async () => {
-    try {
-      await jestPlaywright.resetContext();
-      await page.goto("http://localhost:3000/");
-    } catch (e) {}
+test.describe("run tests independently", () => {
+  /**
+   * @type {Page}
+   */
+  global.page = null;
+  test.beforeEach(async ({ page: _page }) => {
+    global.page = _page;
+    await page.goto("http://localhost:3000/");
   });
 
-  beforeEach(async () => {
-    // we only close and open new page when running tests independently
-    // otherwise we simply navigate to home.
-    if (type === "independent") {
-      if (page.isClosed()) page = await browser.newPage();
-      await page.goto("http://localhost:3000/");
-    } else {
-      // only navigate to Home if we are not at home
-      if ((await page.textContent(getTestId("routeHeader"))) !== "Notes")
-        await navigateTo("notes");
-    }
-  }, 600000);
-
-  // we have to reset the createNoteAndCheckPresence after every test
-  afterEach(async () => {
-    if (type === "independent") {
-      await page.close();
-      createNoteAndCheckPresence = staticCreateNoteAndCheckPresence;
-    } else {
-      createNoteAndCheckPresence = async function () {
-        let noteSelector = List.new("note").atIndex(0).grouped().build();
-        await page.click(noteSelector, { button: "left" });
-        return noteSelector;
-      };
-    }
+  test("create a note", async () => {
+    await createNoteAndCheckPresence();
   });
 
-  test("create a note", createNoteAndCheckPresence);
-
-  test("delete a note", deleteNoteAndCheckAbsence);
+  test("delete a note", async () => {
+    await deleteNoteAndCheckAbsence();
+  });
 
   test("restore a note", async () => {
-    const trashItemSelector =
-      type === "independent"
-        ? await deleteNoteAndCheckAbsence()
-        : List.new("trash").atIndex(0).title().build();
-
-    if (type === "sequential") await navigateTo("trash");
+    const trashItemSelector = await deleteNoteAndCheckAbsence();
 
     await openContextMenu(trashItemSelector);
 
@@ -234,15 +182,15 @@ describe.each(["independent", "sequential"])("run tests %sly", (type) => {
     await checkNotePresence();
   });
 
-  test.skip("add a note to notebook", async () => {
-    const noteSelector = await createNoteAndCheckPresence();
+  // test.skip("add a note to notebook", async () => {
+  //   const noteSelector = await createNoteAndCheckPresence();
 
-    await openContextMenu(noteSelector);
+  //   await openContextMenu(noteSelector);
 
-    await clickMenuItem("addtonotebook(s)");
+  //   await clickMenuItem("addtonotebook(s)");
 
-    await addNoteToNotebook();
-  });
+  //   await addNoteToNotebook();
+  // });
 
   test("favorite a note", async () => {
     const noteSelector = await createNoteAndCheckPresence();
@@ -267,11 +215,9 @@ describe.each(["independent", "sequential"])("run tests %sly", (type) => {
   test("unfavorite a note", async () => {
     const noteSelector = await createNoteAndCheckPresence();
 
-    if (type === "independent") {
-      await useContextMenu(noteSelector, async () => {
-        await clickMenuItem("favorite");
-      });
-    }
+    await useContextMenu(noteSelector, async () => {
+      await clickMenuItem("favorite");
+    });
 
     await page.waitForTimeout(500);
 
@@ -337,8 +283,7 @@ describe.each(["independent", "sequential"])("run tests %sly", (type) => {
   test("unpin a note", async () => {
     const noteSelector = await createNoteAndCheckPresence();
 
-    if (type === "independent")
-      await useContextMenu(noteSelector, () => clickMenuItem("pin"));
+    await useContextMenu(noteSelector, () => clickMenuItem("pin"));
 
     await page.waitForTimeout(500);
 
@@ -377,21 +322,6 @@ describe.each(["independent", "sequential"])("run tests %sly", (type) => {
 
     await expect(page.$(trashItemSelector)).resolves.toBeFalsy();
   });
-});
-
-describe("run tests only independently", () => {
-  beforeAll(() => {
-    createNoteAndCheckPresence = staticCreateNoteAndCheckPresence;
-  });
-
-  beforeEach(async () => {
-    await page.goto("http://localhost:3000/");
-  }, 600000);
-
-  afterEach(async () => {
-    await page.close();
-    page = await browser.newPage();
-  });
 
   test("lock a note", async () => {
     const noteSelector = await createNoteAndCheckPresence();
@@ -406,7 +336,7 @@ describe("run tests only independently", () => {
 
     await lockUnlockNote(noteSelector, "lock");
 
-    await page.waitForTimeout(500);
+    await page.waitForTimeout(1000);
 
     await lockUnlockNote(noteSelector, "unlock");
 
@@ -444,15 +374,15 @@ describe("run tests only independently", () => {
     await checkNoteLocked(noteSelector);
   });
 
-  test.skip("add a note to notebook from properties", async () => {
-    await createNoteAndCheckPresence();
+  // test.skip("add a note to notebook from properties", async () => {
+  //   await createNoteAndCheckPresence();
 
-    await page.click(getTestId("properties"));
+  //   await page.click(getTestId("properties"));
 
-    await page.click(getTestId("properties-add-to-nb"));
+  //   await page.click(getTestId("properties-add-to-nb"));
 
-    await addNoteToNotebook();
-  });
+  //   await addNoteToNotebook();
+  // });
 
   test("assign a color to note from properties", async () => {
     const noteSelector = await createNoteAndCheckPresence();

apps/web/__e2e__/utils/conditions.js

@@ -1,5 +1,6 @@
 /* eslint-disable no-undef */
 
+const { expect } = require("@playwright/test");
 const { getTestId, NOTE } = require(".");
 const List = require("./listitemidbuilder");
 

apps/web/package.json

@@ -47,12 +47,12 @@
     "zustand": "^3.3.1"
   },
   "devDependencies": {
+    "@playwright/test": "^1.12.0",
     "@types/hookrouter": "^2.2.5",
-    "@types/jest": "^26.0.20",
-    "@types/jest-image-snapshot": "^4.1.0",
     "@types/node-fetch": "^2.5.10",
     "@types/quill": "^2.0.5",
     "babel-eslint": "^10.1.0",
+    "concurrently": "^6.2.0",
     "chalk": "^4.1.0",
     "env-cmd": "^10.1.0",
     "eslint": "^7.20.0",
@@ -62,8 +62,8 @@
     "eslint-plugin-jsx-a11y": "^6.2.3",
     "eslint-plugin-react": "^7.22.0",
     "eslint-plugin-react-hooks": "^4.2.0",
-    "jest-image-snapshot": "^4.3.0",
-    "jest-playwright-preset": "^1.4.5",
+    "jest-image-snapshot": "^4.5.0",
+    "npm-run-all": "^4.1.5",
     "progress-bar-webpack-plugin": "^2.1.0",
     "source-map-explorer": "^2.5.2",
     "typescript": "^4.1.5",
@@ -75,7 +75,7 @@
     "build:desktop": "env-cmd -e all,desktop react-scripts build",
     "deploy": "./scripts/deploy.sh",
     "debug": "env-cmd -e all,dev,web,silent react-scripts start",
-    "test": "env-cmd -e dev,silent jest -c jest.e2e.config.js",
+    "test": "node ./scripts/run-tests.js",
     "test:debug": "env-cmd -e dev,debug,silent jest -c jest.e2e.config.js",
     "eject": "react-scripts eject",
     "update": "npm i @streetwriters/editor@latest @streetwriters/notesnook-core@latest @streetwriters/theme@latest",

apps/web/playwright.config.js

@@ -0,0 +1,35 @@
+const { devices } = require("@playwright/test");
+const IS_CI = !!process.env.CI;
+
+module.exports = {
+  // Look for test files in the "tests" directory, relative to this configuration file
+  testDir: "__e2e__",
+
+  // Each test is given 30 seconds
+  timeout: 30000,
+  workers: IS_CI ? 2 : 4,
+  use: {
+    headless: true,
+
+    // Artifacts
+    screenshot: "only-on-failure",
+    video: "retry-with-video",
+  },
+  projects: [
+    {
+      name: "Chromium",
+      use: {
+        // Configure the browser to use.
+        browserName: "chromium",
+      },
+    },
+    {
+      name: "Firefox",
+      use: { browserName: "firefox" },
+    },
+    {
+      name: "WebKit",
+      use: { browserName: "webkit" },
+    },
+  ],
+};

apps/web/scripts/run-tests.js

@@ -0,0 +1,37 @@
+const { spawn } = require("child_process");
+
+const PORT = 3000;
+function startServer() {
+  return new Promise((resolve) => {
+    const process = spawn("yarn", ["debug", "-p", PORT], {
+      detached: true,
+      stdio: "pipe",
+    });
+    process.stdout.on("data", (data) => {
+      const message = data.toString();
+      if (message.includes("create a production build, use yarn build."))
+        resolve(process);
+    });
+  });
+}
+
+function startTestRunner() {
+  return new Promise((resolve) => {
+    const process = spawn("yarn", ["playwright", "test"], {
+      detached: true,
+      stdio: "inherit",
+    });
+    process.on("close", () => resolve());
+  });
+}
+
+(async function () {
+  console.log("Starting server at port", PORT, "...");
+  const server = await startServer();
+
+  console.log("Starting tests...");
+  await startTestRunner();
+
+  console.log("All done.");
+  process.kill(-server.pid);
+})();