feat: migrate from delta to tiny

packages/core/__tests__/backup.test.js

@@ -100,6 +100,11 @@ describe.each([
       ).toBeTruthy();
 
       expect(db.notebooks.all.every((v) => v.title != null)).toBeTruthy();
+
+      const allContent = await db.content.all();
+      expect(
+        allContent.every((v) => v.type === "tiny" || v.deleted)
+      ).toBeTruthy();
     });
   });
 

packages/core/__tests__/lookup.test.js

@@ -14,10 +14,12 @@ beforeEach(async () => {
   StorageInterface.clear();
 });
 
+const content = { ...TEST_NOTE.content, data: "<p>5</p>" };
+
 //TODO
 test("search notes", () =>
   noteTest({
-    content: { type: "delta", data: [{ insert: "5" }] },
+    content: content,
   }).then(async ({ db }) => {
     await db.notes.add(TEST_NOTE);
     let filtered = await db.lookup.notes(db.notes.all, "5");
@@ -26,7 +28,7 @@ test("search notes", () =>
 
 test("search notes with a locked note", () =>
   noteTest({
-    content: { type: "delta", data: [{ insert: "5" }] },
+    content: content,
   }).then(async ({ db }) => {
     const noteId = await db.notes.add(TEST_NOTE);
     await db.vault.create("password");
@@ -37,11 +39,11 @@ test("search notes with a locked note", () =>
 
 test("search notes with an empty note", () =>
   noteTest({
-    content: { type: "delta", data: [{ insert: "5" }] },
+    content: content,
   }).then(async ({ db }) => {
     await db.notes.add({
       title: "hello world",
-      content: { type: "delta", data: [{ insert: "\n" }] },
+      content: { type: "tiny", data: "<p><br></p>" },
     });
     let filtered = await db.lookup.notes(db.notes.all, "hello world");
     expect(filtered.length).toBe(1);

packages/core/__tests__/notes.test.js

@@ -60,7 +60,7 @@ test("get all notes", () =>
 test("note without a title should get title from content", () =>
   noteTest().then(async ({ db, id }) => {
     let note = db.notes.note(id);
-    expect(note.title).toBe("Hello This is colorful");
+    expect(note.title).toBe("HelloThis is colorful");
   }));
 
 test("note title should allow trailing space", () =>
@@ -85,8 +85,8 @@ test("update note", () =>
       id,
       title: "I am a new title",
       content: {
-        type: "delta",
+        type: TEST_NOTE.content.type,
-        data: [],
+        data: "<p><br></p>",
       },
       pinned: true,
       favorite: true,
@@ -106,8 +106,8 @@ test("updating empty note should delete it", () =>
       id,
       title: "\n",
       content: {
-        type: "delta",
+        type: TEST_NOTE.content.type,
-        data: [{ insert: "\n" }],
+        data: "<p><br></p>",
       },
     });
     expect(id).toBeUndefined();
@@ -221,7 +221,7 @@ test("export note to html", () =>
     const html = await db.notes.note(id).export("html");
     expect(
       html.includes(
-        `<p>Hello<br/><span style="color:#f00">This is colorful</span></p>`
+        `<p>Hello<br/><span style='color:#f00'>This is colorful</span></p>`
       )
     ).toBeTruthy();
   }));
@@ -229,7 +229,7 @@ test("export note to html", () =>
 test("export note to md", () =>
   noteTest().then(async ({ db, id }) => {
     const md = await db.notes.note(id).export("md");
-    expect(md.includes(`Hello\nThis is colorful\n`)).toBeTruthy();
+    expect(md.includes(`Hello  \nThis is colorful\n`)).toBeTruthy();
   }));
 
 test("export note to txt", () =>

packages/core/__tests__/topics.test.js

@@ -69,7 +69,7 @@ test("get topic", () =>
     await topics.add("Home");
     let topic = topics.topic("Home");
     let noteId = await db.notes.add({
-      content: { type: "delta", data: [{ insert: "Hello" }] },
+      content: TEST_NOTE.content,
     });
     await topic.add(noteId);
     topic = topics.topic("Home");

packages/core/__tests__/utils/index.js

@@ -27,11 +27,8 @@ const notebookTest = (notebook = TEST_NOTEBOOK) =>
 
 var TEST_NOTE = {
   content: {
-    type: "delta",
+    type: "tiny",
-    data: [
+    data: `<p>Hello<br/><span style='color:#f00'>This is colorful</span></p>`,
-      { insert: "Hello\n" },
-      { insert: "This is colorful", attributes: { color: "#f00" } },
-    ],
   },
 };
 

packages/core/__tests__/vault.test.js

@@ -61,7 +61,7 @@ test("unlock a note", () =>
     const note = await db.vault.open(id, "password");
     expect(note.id).toBe(id);
     expect(note.content.data).toBeDefined();
-    expect(note.content.type).toBe("delta");
+    expect(note.content.type).toBe(TEST_NOTE.content.type);
   }));
 
 test("unlock a note permanently", () =>
@@ -74,7 +74,7 @@ test("unlock a note permanently", () =>
     expect(note.headline).not.toBe("");
     const content = await db.content.raw(note.data.contentId);
     expect(content.data).toBeDefined();
-    expect(typeof content.data).toBe("object");
+    expect(typeof content.data).toBe("string");
   }));
 
 test("save a locked note", () =>

packages/core/api/migrations.js

@@ -49,6 +49,7 @@ class Migrations {
       {
         index: [this._db.settings.raw],
         dbCollection: this._db.settings,
+        type: "settings",
       },
     ];
     await this._migrator.migrate(collections, (item) => item, this.dbVersion);

packages/core/api/sync/__tests__/sync.test.js

@@ -47,7 +47,7 @@ test("sync without merge conflicts, cause merge conflicts, resolve them and then
     // 4. edit the note's content
     await db.notes.add({
       id: noteId,
-      content: { type: "delta", data: [{ insert: "text" }] },
+      content: { ...TEST_NOTE.content },
     });
 
     // 5. sync again and expect conflicts
@@ -57,10 +57,10 @@ test("sync without merge conflicts, cause merge conflicts, resolve them and then
       v: CURRENT_DATABASE_VERSION,
       ...(await getEncrypted({
         id: contentId,
-        type: "delta",
+        type: TEST_NOTE.content.type,
         dateEdited: Date.now(),
         conflicted: false,
-        data: [{ insert: "text" }],
+        data: TEST_NOTE.content.data,
       })),
     };
 
@@ -88,8 +88,8 @@ test("sync without merge conflicts, cause merge conflicts, resolve them and then
       id: noteId,
       conflicted: false,
       content: {
-        type: "delta",
+        type: TEST_NOTE.content.type,
-        data: [{ insert: "text" }],
+        data: TEST_NOTE.content.data,
         resolved: true,
       },
     });

packages/core/common.js

@@ -31,4 +31,4 @@ export const EVENTS = {
   noteRemoved: "note:removed",
 };
 
-export const CURRENT_DATABASE_VERSION = 4.3;
+export const CURRENT_DATABASE_VERSION = 5.0;

packages/core/content-types/delta.js

@@ -1,68 +0,0 @@
-import { QuillDeltaToHtmlConverter } from "quill-delta-to-html";
-import { deltaToMarkdown } from "quill-delta-to-markdown";
-
-const splitter = /\W+/gm;
-class Delta {
-  constructor(data) {
-    this.data = data;
-  }
-
-  toHTML() {
-    const deltaConverter = new QuillDeltaToHtmlConverter(this.data, {
-      classPrefix: "nn",
-      inlineStyles: true,
-    });
-    return deltaConverter.convert();
-  }
-
-  toTXT() {
-    return this.data.reduce(function (text, op) {
-      if (!op.insert) return text;
-      if (typeof op.insert !== "string") return text + " ";
-      return text + op.insert;
-    }, "");
-  }
-
-  toMD() {
-    return deltaToMarkdown(this.data);
-  }
-
-  toTitle() {
-    return getSubstringFromDelta(this.data, 30);
-  }
-
-  toHeadline() {
-    return getSubstringFromDelta(this.data, 80);
-  }
-
-  isEmpty() {
-    return this.toTXT().trim().length <= 0;
-  }
-
-  /**
-   * @returns {Boolean}
-   */
-  search(query) {
-    const tokens = query.toLowerCase().split(splitter);
-    return this.data.some((item) => {
-      if (item.insert && item.insert.indexOf) {
-        return tokens.some(
-          (token) => item.insert.toLowerCase().indexOf(token) > -1
-        );
-      }
-      return false;
-    });
-  }
-}
-export default Delta;
-
-function getSubstringFromDelta(data, limit) {
-  let substr = "";
-  for (var i = 0; i < data.length; ++i) {
-    const item = data[i];
-    if (item.insert && typeof item.insert === "string")
-      substr += item.insert.replace(/\r?\n/g, " ");
-    if (substr.length > limit) return substr.substring(0, limit);
-  }
-  return substr;
-}

packages/core/content-types/index.js

@@ -1,9 +1,9 @@
-import Delta from "./delta";
+import Tiny from "./tiny";
 
 export function getContentFromData(type, data) {
   switch (type) {
-    case "delta":
+    case "tiny":
-      return new Delta(data);
+      return new Tiny(data);
     default:
       return null;
   }

packages/core/content-types/tiny.js

@@ -0,0 +1,48 @@
+import TurndownService from "turndown";
+var turndownService = new TurndownService();
+
+const splitter = /\W+/gm;
+class Tiny {
+  constructor(data) {
+    this.data = data;
+  }
+
+  toHTML() {
+    return this.data;
+  }
+
+  toTXT() {
+    if (window.DOMParser) {
+      let doc = new DOMParser().parseFromString(this.data, "text/html");
+      return doc.body.textContent || "";
+    } else {
+      return this.data.replace(/<br>/gm, "\n").replace(/<[^>]+>/g, "");
+    }
+  }
+
+  toMD() {
+    return turndownService.turndown(this.data);
+  }
+
+  toTitle() {
+    return this.toTXT().substring(0, 30);
+  }
+
+  toHeadline() {
+    return this.toTXT().substring(0, 80);
+  }
+
+  isEmpty() {
+    return this.toTXT().trim().length <= 0;
+  }
+
+  /**
+   * @returns {Boolean}
+   */
+  search(query) {
+    const tokens = query.toLowerCase().split(splitter);
+    const lowercase = this.toTXT().toLowerCase();
+    return tokens.some((token) => lowercase.indexOf(token) > -1);
+  }
+}
+export default Tiny;

packages/core/database/backup.js

@@ -97,6 +97,7 @@ export default class Backup {
       case 4:
       case 4.1:
       case 4.2:
+      case 4.3:
       case 3:
       case 2: {
         return backup;
@@ -144,6 +145,7 @@ export default class Backup {
       {
         index: data["delta"],
         dbCollection: this._db.content,
+        type: "delta",
       },
       {
         index: data["content"],
@@ -152,6 +154,7 @@ export default class Backup {
       {
         index: ["settings"],
         dbCollection: this._db.settings,
+        type: "settings",
       },
     ];
 

packages/core/database/migrator.js

@@ -15,7 +15,7 @@ class Migrator {
             if (item.deleted)
               return await collection.dbCollection._collection.addItem(item);
 
-            const migrate = migrations[version][item.type || id];
+            const migrate = migrations[version][item.type || collection.type];
             if (migrate) item = migrate(item);
 
             if (!!collection.dbCollection.merge) {

packages/core/migrations.js

@@ -1,3 +1,5 @@
+import { QuillDeltaToHtmlConverter } from "quill-delta-to-html";
+
 export const migrations = {
   0: {
     note: function (item) {
@@ -14,7 +16,7 @@ export const migrations = {
       item.data = item.data.ops;
       item.type = "delta";
       item.migrated = true;
-      return item;
+      return migrations["2"].delta(item);
     },
     trash: function (item) {
       item.itemType = item.type;
@@ -40,6 +42,7 @@ export const migrations = {
 
       return migrations[3].note(item);
     },
+    delta: (item) => migrations["3"].delta(item),
   },
   3: {
     note: function (item) {
@@ -49,6 +52,7 @@ export const migrations = {
 
       return migrations[4].note(item);
     },
+    delta: (item) => migrations["4"].delta(item),
   },
   4: {
     note: function (item) {
@@ -57,11 +61,13 @@ export const migrations = {
       }
       return migrations["4.1"].note(item);
     },
+    delta: (item) => migrations["4.1"].delta(item),
   },
   4.1: {
     note: function (item) {
       return migrations["4.2"].note(item);
     },
+    delta: (item) => migrations["4.2"].delta(item),
   },
   4.2: {
     note: function (item) {
@@ -73,8 +79,21 @@ export const migrations = {
       item.migrated = true;
       return item;
     },
+    delta: (item) => migrations["4.3"].delta(item),
   },
   4.3: {
+    delta: function (item) {
+      const deltaConverter = new QuillDeltaToHtmlConverter(item.data, {
+        classPrefix: "nn",
+        inlineStyles: true,
+      });
+      item.data = deltaConverter.convert();
+      item.type = "tiny";
+      item.migrated = true;
+      return item;
+    },
+  },
+  5.0: {
     note: false,
     notebook: false,
     tag: false,

packages/core/package.json

@@ -28,6 +28,7 @@
     "qclone": "^1.0.4",
     "quill-delta-to-html": "^0.12.0",
     "quill-delta-to-markdown": "https://github.com/streetwriters/quill-delta-to-markdown",
-    "transfun": "^1.0.2"
+    "transfun": "^1.0.2",
+    "turndown": "^7.0.0"
   }
 }

packages/core/yarn.lock

@@ -2237,6 +2237,11 @@ domexception@^1.0.1:
   dependencies:
     webidl-conversions "^4.0.2"
 
+domino@^2.1.6:
+  version "2.1.6"
+  resolved "https://registry.yarnpkg.com/domino/-/domino-2.1.6.tgz#fe4ace4310526e5e7b9d12c7de01b7f485a57ffe"
+  integrity sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ==
+
 ecc-jsbn@~0.1.1:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9"
@@ -4782,6 +4787,13 @@ tunnel-agent@^0.6.0:
   dependencies:
     safe-buffer "^5.0.1"
 
+turndown@^7.0.0:
+  version "7.0.0"
+  resolved "https://registry.yarnpkg.com/turndown/-/turndown-7.0.0.tgz#19b2a6a2d1d700387a1e07665414e4af4fec5225"
+  integrity sha512-G1FfxfR0mUNMeGjszLYl3kxtopC4O9DRRiMlMDDVHvU1jaBkGFg4qxIyjIk2aiKLHyDyZvZyu4qBO2guuYBy3Q==
+  dependencies:
+    domino "^2.1.6"
+
 tweetnacl@^0.14.3, tweetnacl@~0.14.0:
   version "0.14.5"
   resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64"