diff --git a/packages/core/__tests__/backup.test.js b/packages/core/__tests__/backup.test.js index f946bb6a8..95b6d072a 100644 --- a/packages/core/__tests__/backup.test.js +++ b/packages/core/__tests__/backup.test.js @@ -100,6 +100,11 @@ describe.each([ ).toBeTruthy(); expect(db.notebooks.all.every((v) => v.title != null)).toBeTruthy(); + + const allContent = await db.content.all(); + expect( + allContent.every((v) => v.type === "tiny" || v.deleted) + ).toBeTruthy(); }); }); diff --git a/packages/core/__tests__/lookup.test.js b/packages/core/__tests__/lookup.test.js index f0d110a1c..a74c0d25a 100644 --- a/packages/core/__tests__/lookup.test.js +++ b/packages/core/__tests__/lookup.test.js @@ -14,10 +14,12 @@ beforeEach(async () => { StorageInterface.clear(); }); +const content = { ...TEST_NOTE.content, data: "

5

" }; + //TODO test("search notes", () => noteTest({ - content: { type: "delta", data: [{ insert: "5" }] }, + content: content, }).then(async ({ db }) => { await db.notes.add(TEST_NOTE); let filtered = await db.lookup.notes(db.notes.all, "5"); @@ -26,7 +28,7 @@ test("search notes", () => test("search notes with a locked note", () => noteTest({ - content: { type: "delta", data: [{ insert: "5" }] }, + content: content, }).then(async ({ db }) => { const noteId = await db.notes.add(TEST_NOTE); await db.vault.create("password"); @@ -37,11 +39,11 @@ test("search notes with a locked note", () => test("search notes with an empty note", () => noteTest({ - content: { type: "delta", data: [{ insert: "5" }] }, + content: content, }).then(async ({ db }) => { await db.notes.add({ title: "hello world", - content: { type: "delta", data: [{ insert: "\n" }] }, + content: { type: "tiny", data: "


" }, }); let filtered = await db.lookup.notes(db.notes.all, "hello world"); expect(filtered.length).toBe(1); diff --git a/packages/core/__tests__/notes.test.js b/packages/core/__tests__/notes.test.js index 71a43360f..35ad6b41b 100644 --- a/packages/core/__tests__/notes.test.js +++ b/packages/core/__tests__/notes.test.js @@ -60,7 +60,7 @@ test("get all notes", () => test("note without a title should get title from content", () => noteTest().then(async ({ db, id }) => { let note = db.notes.note(id); - expect(note.title).toBe("Hello This is colorful"); + expect(note.title).toBe("HelloThis is colorful"); })); test("note title should allow trailing space", () => @@ -85,8 +85,8 @@ test("update note", () => id, title: "I am a new title", content: { - type: "delta", - data: [], + type: TEST_NOTE.content.type, + data: "


", }, pinned: true, favorite: true, @@ -106,8 +106,8 @@ test("updating empty note should delete it", () => id, title: "\n", content: { - type: "delta", - data: [{ insert: "\n" }], + type: TEST_NOTE.content.type, + data: "


", }, }); expect(id).toBeUndefined(); @@ -221,7 +221,7 @@ test("export note to html", () => const html = await db.notes.note(id).export("html"); expect( html.includes( - `

Hello
This is colorful

` + `

Hello
This is colorful

` ) ).toBeTruthy(); })); @@ -229,7 +229,7 @@ test("export note to html", () => test("export note to md", () => noteTest().then(async ({ db, id }) => { const md = await db.notes.note(id).export("md"); - expect(md.includes(`Hello\nThis is colorful\n`)).toBeTruthy(); + expect(md.includes(`Hello \nThis is colorful\n`)).toBeTruthy(); })); test("export note to txt", () => diff --git a/packages/core/__tests__/topics.test.js b/packages/core/__tests__/topics.test.js index ecc6d8076..aa5bd7053 100644 --- a/packages/core/__tests__/topics.test.js +++ b/packages/core/__tests__/topics.test.js @@ -69,7 +69,7 @@ test("get topic", () => await topics.add("Home"); let topic = topics.topic("Home"); let noteId = await db.notes.add({ - content: { type: "delta", data: [{ insert: "Hello" }] }, + content: TEST_NOTE.content, }); await topic.add(noteId); topic = topics.topic("Home"); diff --git a/packages/core/__tests__/utils/index.js b/packages/core/__tests__/utils/index.js index 662e09738..00db71e4b 100644 --- a/packages/core/__tests__/utils/index.js +++ b/packages/core/__tests__/utils/index.js @@ -27,11 +27,8 @@ const notebookTest = (notebook = TEST_NOTEBOOK) => var TEST_NOTE = { content: { - type: "delta", - data: [ - { insert: "Hello\n" }, - { insert: "This is colorful", attributes: { color: "#f00" } }, - ], + type: "tiny", + data: `

Hello
This is colorful

`, }, }; diff --git a/packages/core/__tests__/vault.test.js b/packages/core/__tests__/vault.test.js index 5f2d93384..21223e3d0 100644 --- a/packages/core/__tests__/vault.test.js +++ b/packages/core/__tests__/vault.test.js @@ -61,7 +61,7 @@ test("unlock a note", () => const note = await db.vault.open(id, "password"); expect(note.id).toBe(id); expect(note.content.data).toBeDefined(); - expect(note.content.type).toBe("delta"); + expect(note.content.type).toBe(TEST_NOTE.content.type); })); test("unlock a note permanently", () => @@ -74,7 +74,7 @@ test("unlock a note permanently", () => expect(note.headline).not.toBe(""); const content = await db.content.raw(note.data.contentId); expect(content.data).toBeDefined(); - expect(typeof content.data).toBe("object"); + expect(typeof content.data).toBe("string"); })); test("save a locked note", () => diff --git a/packages/core/api/migrations.js b/packages/core/api/migrations.js index 9715d425d..6a7bf79a1 100644 --- a/packages/core/api/migrations.js +++ b/packages/core/api/migrations.js @@ -49,6 +49,7 @@ class Migrations { { index: [this._db.settings.raw], dbCollection: this._db.settings, + type: "settings", }, ]; await this._migrator.migrate(collections, (item) => item, this.dbVersion); diff --git a/packages/core/api/sync/__tests__/sync.test.js b/packages/core/api/sync/__tests__/sync.test.js index 44922c5cc..d0066a6f4 100644 --- a/packages/core/api/sync/__tests__/sync.test.js +++ b/packages/core/api/sync/__tests__/sync.test.js @@ -47,7 +47,7 @@ test("sync without merge conflicts, cause merge conflicts, resolve them and then // 4. edit the note's content await db.notes.add({ id: noteId, - content: { type: "delta", data: [{ insert: "text" }] }, + content: { ...TEST_NOTE.content }, }); // 5. sync again and expect conflicts @@ -57,10 +57,10 @@ test("sync without merge conflicts, cause merge conflicts, resolve them and then v: CURRENT_DATABASE_VERSION, ...(await getEncrypted({ id: contentId, - type: "delta", + type: TEST_NOTE.content.type, dateEdited: Date.now(), conflicted: false, - data: [{ insert: "text" }], + data: TEST_NOTE.content.data, })), }; @@ -88,8 +88,8 @@ test("sync without merge conflicts, cause merge conflicts, resolve them and then id: noteId, conflicted: false, content: { - type: "delta", - data: [{ insert: "text" }], + type: TEST_NOTE.content.type, + data: TEST_NOTE.content.data, resolved: true, }, }); diff --git a/packages/core/common.js b/packages/core/common.js index 8ff452358..e1dda4653 100644 --- a/packages/core/common.js +++ b/packages/core/common.js @@ -31,4 +31,4 @@ export const EVENTS = { noteRemoved: "note:removed", }; -export const CURRENT_DATABASE_VERSION = 4.3; +export const CURRENT_DATABASE_VERSION = 5.0; diff --git a/packages/core/content-types/delta.js b/packages/core/content-types/delta.js deleted file mode 100644 index 63ce6481d..000000000 --- a/packages/core/content-types/delta.js +++ /dev/null @@ -1,68 +0,0 @@ -import { QuillDeltaToHtmlConverter } from "quill-delta-to-html"; -import { deltaToMarkdown } from "quill-delta-to-markdown"; - -const splitter = /\W+/gm; -class Delta { - constructor(data) { - this.data = data; - } - - toHTML() { - const deltaConverter = new QuillDeltaToHtmlConverter(this.data, { - classPrefix: "nn", - inlineStyles: true, - }); - return deltaConverter.convert(); - } - - toTXT() { - return this.data.reduce(function (text, op) { - if (!op.insert) return text; - if (typeof op.insert !== "string") return text + " "; - return text + op.insert; - }, ""); - } - - toMD() { - return deltaToMarkdown(this.data); - } - - toTitle() { - return getSubstringFromDelta(this.data, 30); - } - - toHeadline() { - return getSubstringFromDelta(this.data, 80); - } - - isEmpty() { - return this.toTXT().trim().length <= 0; - } - - /** - * @returns {Boolean} - */ - search(query) { - const tokens = query.toLowerCase().split(splitter); - return this.data.some((item) => { - if (item.insert && item.insert.indexOf) { - return tokens.some( - (token) => item.insert.toLowerCase().indexOf(token) > -1 - ); - } - return false; - }); - } -} -export default Delta; - -function getSubstringFromDelta(data, limit) { - let substr = ""; - for (var i = 0; i < data.length; ++i) { - const item = data[i]; - if (item.insert && typeof item.insert === "string") - substr += item.insert.replace(/\r?\n/g, " "); - if (substr.length > limit) return substr.substring(0, limit); - } - return substr; -} diff --git a/packages/core/content-types/index.js b/packages/core/content-types/index.js index dee0f518b..3d8441400 100644 --- a/packages/core/content-types/index.js +++ b/packages/core/content-types/index.js @@ -1,9 +1,9 @@ -import Delta from "./delta"; +import Tiny from "./tiny"; export function getContentFromData(type, data) { switch (type) { - case "delta": - return new Delta(data); + case "tiny": + return new Tiny(data); default: return null; } diff --git a/packages/core/content-types/tiny.js b/packages/core/content-types/tiny.js new file mode 100644 index 000000000..e0aa3e36e --- /dev/null +++ b/packages/core/content-types/tiny.js @@ -0,0 +1,48 @@ +import TurndownService from "turndown"; +var turndownService = new TurndownService(); + +const splitter = /\W+/gm; +class Tiny { + constructor(data) { + this.data = data; + } + + toHTML() { + return this.data; + } + + toTXT() { + if (window.DOMParser) { + let doc = new DOMParser().parseFromString(this.data, "text/html"); + return doc.body.textContent || ""; + } else { + return this.data.replace(/
/gm, "\n").replace(/<[^>]+>/g, ""); + } + } + + toMD() { + return turndownService.turndown(this.data); + } + + toTitle() { + return this.toTXT().substring(0, 30); + } + + toHeadline() { + return this.toTXT().substring(0, 80); + } + + isEmpty() { + return this.toTXT().trim().length <= 0; + } + + /** + * @returns {Boolean} + */ + search(query) { + const tokens = query.toLowerCase().split(splitter); + const lowercase = this.toTXT().toLowerCase(); + return tokens.some((token) => lowercase.indexOf(token) > -1); + } +} +export default Tiny; diff --git a/packages/core/database/backup.js b/packages/core/database/backup.js index f09b891bd..dce92ddab 100644 --- a/packages/core/database/backup.js +++ b/packages/core/database/backup.js @@ -97,6 +97,7 @@ export default class Backup { case 4: case 4.1: case 4.2: + case 4.3: case 3: case 2: { return backup; @@ -144,6 +145,7 @@ export default class Backup { { index: data["delta"], dbCollection: this._db.content, + type: "delta", }, { index: data["content"], @@ -152,6 +154,7 @@ export default class Backup { { index: ["settings"], dbCollection: this._db.settings, + type: "settings", }, ]; diff --git a/packages/core/database/migrator.js b/packages/core/database/migrator.js index cae683e8e..76df21270 100644 --- a/packages/core/database/migrator.js +++ b/packages/core/database/migrator.js @@ -15,7 +15,7 @@ class Migrator { if (item.deleted) return await collection.dbCollection._collection.addItem(item); - const migrate = migrations[version][item.type || id]; + const migrate = migrations[version][item.type || collection.type]; if (migrate) item = migrate(item); if (!!collection.dbCollection.merge) { diff --git a/packages/core/migrations.js b/packages/core/migrations.js index 6d3480b1a..44e191fa2 100644 --- a/packages/core/migrations.js +++ b/packages/core/migrations.js @@ -1,3 +1,5 @@ +import { QuillDeltaToHtmlConverter } from "quill-delta-to-html"; + export const migrations = { 0: { note: function (item) { @@ -14,7 +16,7 @@ export const migrations = { item.data = item.data.ops; item.type = "delta"; item.migrated = true; - return item; + return migrations["2"].delta(item); }, trash: function (item) { item.itemType = item.type; @@ -40,6 +42,7 @@ export const migrations = { return migrations[3].note(item); }, + delta: (item) => migrations["3"].delta(item), }, 3: { note: function (item) { @@ -49,6 +52,7 @@ export const migrations = { return migrations[4].note(item); }, + delta: (item) => migrations["4"].delta(item), }, 4: { note: function (item) { @@ -57,11 +61,13 @@ export const migrations = { } return migrations["4.1"].note(item); }, + delta: (item) => migrations["4.1"].delta(item), }, 4.1: { note: function (item) { return migrations["4.2"].note(item); }, + delta: (item) => migrations["4.2"].delta(item), }, 4.2: { note: function (item) { @@ -73,8 +79,21 @@ export const migrations = { item.migrated = true; return item; }, + delta: (item) => migrations["4.3"].delta(item), }, 4.3: { + delta: function (item) { + const deltaConverter = new QuillDeltaToHtmlConverter(item.data, { + classPrefix: "nn", + inlineStyles: true, + }); + item.data = deltaConverter.convert(); + item.type = "tiny"; + item.migrated = true; + return item; + }, + }, + 5.0: { note: false, notebook: false, tag: false, diff --git a/packages/core/package.json b/packages/core/package.json index 3c9f104ff..f240e976a 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -28,6 +28,7 @@ "qclone": "^1.0.4", "quill-delta-to-html": "^0.12.0", "quill-delta-to-markdown": "https://github.com/streetwriters/quill-delta-to-markdown", - "transfun": "^1.0.2" + "transfun": "^1.0.2", + "turndown": "^7.0.0" } } diff --git a/packages/core/yarn.lock b/packages/core/yarn.lock index 741a94d9a..6bd950e25 100644 --- a/packages/core/yarn.lock +++ b/packages/core/yarn.lock @@ -2237,6 +2237,11 @@ domexception@^1.0.1: dependencies: webidl-conversions "^4.0.2" +domino@^2.1.6: + version "2.1.6" + resolved "https://registry.yarnpkg.com/domino/-/domino-2.1.6.tgz#fe4ace4310526e5e7b9d12c7de01b7f485a57ffe" + integrity sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ== + ecc-jsbn@~0.1.1: version "0.1.2" resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" @@ -4782,6 +4787,13 @@ tunnel-agent@^0.6.0: dependencies: safe-buffer "^5.0.1" +turndown@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/turndown/-/turndown-7.0.0.tgz#19b2a6a2d1d700387a1e07665414e4af4fec5225" + integrity sha512-G1FfxfR0mUNMeGjszLYl3kxtopC4O9DRRiMlMDDVHvU1jaBkGFg4qxIyjIk2aiKLHyDyZvZyu4qBO2guuYBy3Q== + dependencies: + domino "^2.1.6" + tweetnacl@^0.14.3, tweetnacl@~0.14.0: version "0.14.5" resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64"