From 630bb33b10983b965e2dc9ed2ee21ed9f9221fd0 Mon Sep 17 00:00:00 2001 From: Abdullah Atta Date: Tue, 21 Dec 2021 13:41:08 +0500 Subject: [PATCH] feat: add note version history (#20) * feat: note version history * fix bugs * add tests for session history * update tests * add tests for restoring content * add more tests * update jsdoc * return empty array if no session history present for a note * init note history with await * add note history versions limit * cleanup note history after adding a new session * fix tests * add test for session cleanup * test: add collector tests * feat: dateEdited -> dateModified * feat: migrate to liqe for searching * chore: forceExit jest after test run * feat: note version history * fix bugs * add tests for session history * update tests * add tests for restoring content * add more tests * update jsdoc * return empty array if no session history present for a note * init note history with await * add note history versions limit * cleanup note history after adding a new session * fix tests * add test for session cleanup * init ContentHistory with await * fix reference to db in init * make sessionId in db * check if note is locked through note metadata * make cleanup method private * use remove methods in notehistory and contenthistory * restore note content via notes.add method if note is not locked * move getting all items to seperate functions * check if parsed json is valid * deserialize a session only if it's sessionContent exists * add locked property to sessionContent * remove makeSessionId function * update tests * remove yarn.lock file * update tests Co-authored-by: ammarahm-ed --- packages/core/__tests__/note-history.test.js | 181 ++++++++++++++++ packages/core/api/index.js | 3 + packages/core/collections/contenthistory.js | 54 +++++ packages/core/collections/note-history.js | 213 +++++++++++++++++++ packages/core/package.json | 2 + packages/core/utils/compression.js | 27 +++ packages/core/utils/id.js | 9 + 7 files changed, 489 insertions(+) create mode 100644 packages/core/__tests__/note-history.test.js create mode 100644 packages/core/collections/contenthistory.js create mode 100644 packages/core/collections/note-history.js create mode 100644 packages/core/utils/compression.js diff --git a/packages/core/__tests__/note-history.test.js b/packages/core/__tests__/note-history.test.js new file mode 100644 index 000000000..2d8fbd4ab --- /dev/null +++ b/packages/core/__tests__/note-history.test.js @@ -0,0 +1,181 @@ +import { compress, decompress } from "../utils/compression"; +import { databaseTest, noteTest, StorageInterface, TEST_NOTE } from "./utils"; + +beforeEach(async () => { + StorageInterface.clear(); +}); + + +async function sessionTest(db, noteId) { + let note = await db.notes.note(noteId).data; + let content = { + data: await db.notes.note(noteId).content(), + type: "tiny", + }; + let session = await db.noteHistory.add( + noteId, + note.dateEdited, + content + ); + + return session; +} + +test("History of note should be created", async () => { + let { db, id } = await noteTest(); + let session = await sessionTest(db, id); + let content = { + data: await db.notes.note(id).content(), + type: "tiny", + }; + + let sessionContent = await db.noteHistory.content(session.sessionContentId); + expect(sessionContent).toMatchObject(content); +}); + +test("Multiple sessions of the same note should be created", async () => { + let { db, id } = await noteTest(); + await sessionTest(db, id); + + let nextContent = { + data: (await db.notes.note(id).content().data) + "teststring", + type: "tiny", + }; + + await db.notes.add({ + id: id, + content: nextContent, + }); + let note = db.notes.note(id).data; + await db.noteHistory.add(id, note.dateEdited, nextContent); + + let history = await db.noteHistory.get(id); + expect(history.length).toBe(2); +}); + + +test("Session should be removed if greater than the version limit", async () => { + let { db, id } = await noteTest(); + await sessionTest(db, id); + + let nextContent = { + data: (await db.notes.note(id).content().data) + "teststring", + type: "tiny", + }; + + await db.notes.add({ + id: id, + content: nextContent, + }); + let note = db.notes.note(id).data; + await db.noteHistory.add(id, note.dateEdited, nextContent); + + let history = await db.noteHistory.get(id); + expect(history.length).toBe(2); + await db.noteHistory._cleanup(id,1); + history = await db.noteHistory.get(id); + expect(history.length).toBe(1); + let content = await db.noteHistory.content(history[0].sessionContentId); + expect(content.data).toBe(nextContent.data); + +}); + + + +test("Session should update if a sessionId is same", async () => { + let { db, id } = await noteTest(); + await sessionTest(db, id); + let content = { + data: await db.notes.note(id).content(), + type: "tiny", + }; + let note = db.notes.note(id).data; + let nextContent = { + data: content.data + "teststring", + type: "tiny", + }; + + await db.notes.add({ + id: id, + content: nextContent, + }); + let session = await db.noteHistory.add( + id, + note.dateEdited, + nextContent + ); + let history = await db.noteHistory.get(id); + expect(history.length).toBe(1); + + let sessionContent = await db.noteHistory.content(session.sessionContentId); + expect(sessionContent.data).toBe(nextContent.data); +}); + +test("History of note should be restored", async () => { + let { db, id } = await noteTest(); + let session = await sessionTest(db, id); + let prevContent = { + data: await db.notes.note(id).content(), + type: "tiny", + }; + + await db.notes.add({ + id: id, + content: { + data: "

", + type: "tiny", + }, + }); + await db.noteHistory.restore(session.id); + let nextContent = await db.notes.note(id).content(); + expect(nextContent).toBe(prevContent.data); +}); + +test("Session should not be created if values are falsy", async () => { + let db = await databaseTest(); + let session = await db.noteHistory.add(null, null, null); + expect(session).toBeFalsy(); +}); + +test("Should return empty array if no history available", async () => { + let { db, id } = await noteTest(); + let history = await db.noteHistory.get(id); + expect(history.length).toBe(0); +}); + +test("Session history of a given sessionId should be removed", async () => { + let { db, id } = await noteTest(); + let session = await sessionTest(db, id); + await db.noteHistory.removeSession(session.id); + let history = await db.noteHistory.get(id); + expect(history.length).toBe(0); +}); + +test("All sessions of a note should be cleared", async () => { + let { db, id } = await noteTest(); + await sessionTest(db, id); + + await db.noteHistory.clearSessions(id); + + let history = await db.noteHistory.get(id); + expect(history.length).toBe(0); +}); + +test("Sessions should be serialized and deserialized", async () => { + let { db, id } = await noteTest(); + await sessionTest(db, id); + let json = await db.noteHistory.serialize(); + await db.noteHistory.clearSessions(id); + await db.noteHistory.deserialize(json); + + let history = await db.noteHistory.get(id); + expect(history.length).toBe(1); + let content = await db.noteHistory.content(history[0].id); + expect(content).toBeTruthy(); +}); + +test("String should compress and decompress", () => { + let compressed = compress(TEST_NOTE.content.data); + let decompressed = decompress(compressed); + expect(decompressed).toBe(TEST_NOTE.content.data); +}); diff --git a/packages/core/api/index.js b/packages/core/api/index.js index 2678501a1..811863a96 100644 --- a/packages/core/api/index.js +++ b/packages/core/api/index.js @@ -23,6 +23,7 @@ import Offers from "./offers"; import Attachments from "../collections/attachments"; import Debug from "./debug"; import { Mutex } from "async-mutex"; +import NoteHistory from "../collections/note-history"; /** * @type {EventSource} @@ -109,6 +110,8 @@ class Database { this.content = await Content.new(this, "content", false); /** @type {Attachments} */ this.attachments = await Attachments.new(this, "attachments"); + /**@type {NoteHistory} */ + this.noteHistory = await NoteHistory.new(this,"notehistory"); this.trash = new Trash(this); diff --git a/packages/core/collections/contenthistory.js b/packages/core/collections/contenthistory.js new file mode 100644 index 000000000..a4038feaa --- /dev/null +++ b/packages/core/collections/contenthistory.js @@ -0,0 +1,54 @@ +import { compress, decompress } from "../utils/compression"; +import { makeSessionContentId } from "../utils/id"; +import Collection from "./collection"; + +export default class ContentHistory extends Collection { + /** + * + * @param {string} sessionId + * @param {{content:string:data:string}} content + */ + async add(sessionId, content, locked) { + if (!sessionId || !content) return; + let compressed = locked ? null : compress(content.data); + + await this._collection.addItem({ + id: makeSessionContentId(sessionId), + data: compressed || content.data, + type: content.type, + compressed: !!compressed, + locked + }); + } + + /** + * + * @param {string} sessionId + * @returns {Promise<{content:string;data:string}>} + */ + async get(sessionContentId) { + if (!sessionContentId) return; + let session = await this._collection.getItem(sessionContentId); + return { + data: session.compressed ? decompress(session.data) : session.data, + type: session.type, + }; + } + + /** + * + * @param {string} sessionContentId + */ + async remove(sessionContentId) { + await this._collection.deleteItem(sessionContentId); + } + + + async all() { + let indices = await this._collection.indexer.getIndices(); + let items = + (await this._collection.getItems(indices)); + + return Object.values(items); + } +} diff --git a/packages/core/collections/note-history.js b/packages/core/collections/note-history.js new file mode 100644 index 000000000..c032d1e17 --- /dev/null +++ b/packages/core/collections/note-history.js @@ -0,0 +1,213 @@ +import { makeSessionContentId, makeSessionId } from "../utils/id"; +import Collection from "./collection"; +import ContentHistory from "./contenthistory"; +/** + * @typedef Session + * @property {string} id + * @property {string} noteId + * @property {string} sessionContentId + * @property {string} dateEdited + * @property {string} dateCreated + * @property {boolean} locked + */ + +/** + * @typedef Content + * @property {string} data + * @property {string} type + */ + +export default class NoteHistory extends Collection { + constructor(db, name, cached) { + super(db, name, cached); + this.versionsLimit = 100; + } + + async init() { + super.init(); + + /** + * @type {ContentHistory} + */ + this.contentHistory = await ContentHistory.new( + this._db, + "contenthistory", + false + ); + } + + /** + * Get complete session history of a note. + * @param noteId id of the note + * @returns {Promise} An array of session ids of a note + */ + async get(noteId) { + if (!noteId) return []; + + let indices = await this._collection.indexer.getIndices(); + let sessionIds = indices.filter((id) => id.includes(noteId)); + if (sessionIds.length === 0) return []; + let history = (await this._collection.getItems(sessionIds)) || []; + + return history; + } + + /** + * Add and update a session content + * @param {string} noteId id of the note + * @param {string} dateEdited edited date of the note + * @param {Content} content + * + * @returns {Promise} + */ + async add(noteId, dateEdited, content) { + if (!noteId || !dateEdited || !content) return; + let sessionId = `${noteId}_${dateEdited}`; + let exists = await this._collection.exists(sessionId); + let locked = this._db.notes.note(noteId)?.data?.locked; + + let session = { + id: sessionId, + sessionContentId: makeSessionContentId(sessionId), + noteId, + }; + if (!exists) { + session.dateCreated = Date.now(); + } + + if (locked) { + session.locked = true; + } + + await this._collection.addItem(session); + await this.contentHistory.add(sessionId, content, session.locked); + await this._cleanup(noteId); + + return session; + } + + async _cleanup(noteId, limit = this.versionsLimit) { + let history = await this.get(noteId); + if (history.length === 0 || history.length < limit) return; + history.sort(function (a, b) { + return a.dateEdited - b.dateEdited; + }); + let deleteCount = history.length - limit; + + for (let i = 0; i < deleteCount; i++) { + let session = history[i]; + await this._remove(session); + } + } + + /** + * Get content of a session + * @param {string} sessionId session id + * + * @returns {Promise} + */ + async content(sessionId) { + if (!sessionId) return; + return await this.contentHistory.get(sessionId); + } + + /** + * Remove a session from storage + * @param {string} sessionId + */ + async removeSession(sessionId) { + if (!sessionId) return; + /** + * @type {Session} + */ + let session = this._collection.getItem(sessionId); + await this._remove(session); + } + + /** + * Remove all sessions of a note from storage + * @param {string} noteId + */ + async clearSessions(noteId) { + if (!noteId) return; + let history = await this.get(noteId); + for (let item of history) { + await this._remove(item); + } + } + + /** + * + * @param {Session} session + */ + async _remove(session) { + await this._collection.deleteItem(session.id); + await this.contentHistory.remove(session.sessionContentId); + } + + /** + * + * @param {string} sessionId + */ + async restore(sessionId) { + /** + * @type {Session} + */ + let session = await this._collection.getItem(sessionId); + let content = await this.contentHistory.get(session.sessionContentId); + let note = this._db.notes.note(session.noteId).data; + if (session.locked) { + await this._db.content.add({ + id: note.contentId, + data: content.data, + type: content.type, + }); + } else { + await this._db.notes.add({ + id: session.noteId, + content: { + data: content.data, + type: content.type, + }, + }); + } + } + + /** + * + * @returns A json string containing all sessions with content + */ + async serialize() { + return JSON.stringify({ + sessions: await this.all(), + sessionContents: await this.contentHistory.all(), + }); + } + + async all() { + let indices = await this._collection.indexer.getIndices(); + let items = await this._collection.getItems(indices); + return Object.values(items); + } + + /** + * Restore session history from a serialized json string. + * @param {string} data + * @returns + */ + async deserialize(data) { + if (!data) return; + let deserialized = JSON.parse(data); + if (!deserialized.sessions || !deserialized.sessionContents) return; + + for (let session of deserialized.sessions) { + let sessionContent = deserialized.sessionContents.find((v) => + v.id.includes(session.id) + ); + if (sessionContent) { + await this._collection.addItem(session); + await this.contentHistory._collection.addItem(sessionContent); + } + } + } +} diff --git a/packages/core/package.json b/packages/core/package.json index 6749fc96f..6342c0618 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -25,9 +25,11 @@ "dependencies": { "@stablelib/blake2s": "^1.0.1", "async-mutex": "^0.3.2", + "base64-arraybuffer": "^1.0.1", "dayjs": "^1.10.6", "fast-sort": "^2.0.1", "liqe": "^1.13.0", + "fflate": "^0.7.1", "node-html-parser": "github:thecodrr/node-html-parser", "qclone": "^1.0.4", "showdown": "github:thecodrr/showdown", diff --git a/packages/core/utils/compression.js b/packages/core/utils/compression.js new file mode 100644 index 000000000..8d7c6ec92 --- /dev/null +++ b/packages/core/utils/compression.js @@ -0,0 +1,27 @@ +import { decode, encode } from "base64-arraybuffer"; +import fflate from "fflate"; + + +/** + * + * @param {string} data + * @returns {string | null} An object containing compressed data + */ +export const compress = (data) => { + try { + return encode(fflate.compressSync(fflate.strToU8(data)).buffer) + } catch (e) { + return null; + } +}; + +/** + * + * @param {string} compressed + * @returns {string} decompressed string + */ +export const decompress = (compressed) => { + return fflate.strFromU8( + fflate.decompressSync(new Uint8Array(decode(compressed))) + ); +}; diff --git a/packages/core/utils/id.js b/packages/core/utils/id.js index fd7aaabf3..b6ab69690 100644 --- a/packages/core/utils/id.js +++ b/packages/core/utils/id.js @@ -14,3 +14,12 @@ export default function () { export function makeId(text) { return SparkMD5.hash(text); } + +/** + * + * @param {string} noteId id of a note + * @returns {string} An id with postfix of "_index" + */ +export function makeSessionContentId(sessionId) { + return sessionId + "_content"; +}