feat: add note version history (#20)

* feat: note version history

* fix bugs

* add tests for session history

* update tests

* add tests for restoring content

* add more tests

* update jsdoc

* return empty array if no session history present for a note

* init note history with await

* add note history versions limit

* cleanup note history after adding a new session

* fix tests

* add test for session cleanup

* test: add collector tests

* feat: dateEdited -> dateModified

* feat: migrate to liqe for searching

* chore: forceExit jest after test run

* feat: note version history

* fix bugs

* add tests for session history

* update tests

* add tests for restoring content

* add more tests

* update jsdoc

* return empty array if no session history present for a note

* init note history with await

* add note history versions limit

* cleanup note history after adding a new session

* fix tests

* add test for session cleanup

* init ContentHistory with await

* fix reference to db in init

* make sessionId in db

* check if note is locked through note metadata

* make cleanup method private

* use remove methods in notehistory and contenthistory

* restore note content via notes.add method if note is not locked

* move getting all items to seperate functions

* check if parsed json is valid

* deserialize a session only if it's sessionContent exists

* add locked property to sessionContent

* remove makeSessionId function

* update tests

* remove yarn.lock file

* update tests

Co-authored-by: ammarahm-ed <ammarahmed6506@gmail.com>

packages/core/__tests__/note-history.test.js

@@ -0,0 +1,181 @@
+import { compress, decompress } from "../utils/compression";
+import { databaseTest, noteTest, StorageInterface, TEST_NOTE } from "./utils";
+
+beforeEach(async () => {
+  StorageInterface.clear();
+});
+
+
+async function sessionTest(db, noteId) {
+  let note = await db.notes.note(noteId).data;
+  let content = {
+    data: await db.notes.note(noteId).content(),
+    type: "tiny",
+  };
+  let session = await db.noteHistory.add(
+    noteId,
+    note.dateEdited,
+    content
+  );
+
+  return session;
+}
+
+test("History of note should be created", async () => {
+  let { db, id } = await noteTest();
+  let session = await sessionTest(db, id);
+  let content = {
+    data: await db.notes.note(id).content(),
+    type: "tiny",
+  };
+
+  let sessionContent = await db.noteHistory.content(session.sessionContentId);
+  expect(sessionContent).toMatchObject(content);
+});
+
+test("Multiple sessions of the same note should be created", async () => {
+  let { db, id } = await noteTest();
+  await sessionTest(db, id);
+
+  let nextContent = {
+    data: (await db.notes.note(id).content().data) + "teststring",
+    type: "tiny",
+  };
+
+  await db.notes.add({
+    id: id,
+    content: nextContent,
+  });
+  let note = db.notes.note(id).data;
+  await db.noteHistory.add(id, note.dateEdited, nextContent);
+
+  let history = await db.noteHistory.get(id);
+  expect(history.length).toBe(2);
+});
+
+
+test("Session should be removed if greater than the version limit", async () => {
+  let { db, id } = await noteTest();
+  await sessionTest(db, id);
+
+  let nextContent = {
+    data: (await db.notes.note(id).content().data) + "teststring",
+    type: "tiny",
+  };
+
+  await db.notes.add({
+    id: id,
+    content: nextContent,
+  });
+  let note = db.notes.note(id).data;
+  await db.noteHistory.add(id, note.dateEdited, nextContent);
+
+  let history = await db.noteHistory.get(id);
+  expect(history.length).toBe(2);
+  await db.noteHistory._cleanup(id,1);
+  history = await db.noteHistory.get(id);
+  expect(history.length).toBe(1);
+  let content = await db.noteHistory.content(history[0].sessionContentId);
+  expect(content.data).toBe(nextContent.data);
+
+});
+
+
+
+test("Session should update if a sessionId is same", async () => {
+  let { db, id } = await noteTest();
+  await sessionTest(db, id);
+  let content = {
+    data: await db.notes.note(id).content(),
+    type: "tiny",
+  };
+  let note = db.notes.note(id).data;
+  let nextContent = {
+    data: content.data + "teststring",
+    type: "tiny",
+  };
+
+  await db.notes.add({
+    id: id,
+    content: nextContent,
+  });
+  let session = await db.noteHistory.add(
+    id,
+    note.dateEdited,
+    nextContent
+  );
+  let history = await db.noteHistory.get(id);
+  expect(history.length).toBe(1);
+
+  let sessionContent = await db.noteHistory.content(session.sessionContentId);
+  expect(sessionContent.data).toBe(nextContent.data);
+});
+
+test("History of note should be restored", async () => {
+  let { db, id } = await noteTest();
+  let session = await sessionTest(db, id);
+  let prevContent = {
+    data: await db.notes.note(id).content(),
+    type: "tiny",
+  };
+
+  await db.notes.add({
+    id: id,
+    content: {
+      data: "<p></p>",
+      type: "tiny",
+    },
+  });
+  await db.noteHistory.restore(session.id);
+  let nextContent = await db.notes.note(id).content();
+  expect(nextContent).toBe(prevContent.data);
+});
+
+test("Session should not be created if values are falsy", async () => {
+  let db = await databaseTest();
+  let session = await db.noteHistory.add(null, null, null);
+  expect(session).toBeFalsy();
+});
+
+test("Should return empty array if no history available", async () => {
+  let { db, id } = await noteTest();
+  let history = await db.noteHistory.get(id);
+  expect(history.length).toBe(0);
+});
+
+test("Session history of a given sessionId should be removed", async () => {
+  let { db, id } = await noteTest();
+  let session = await sessionTest(db, id);
+  await db.noteHistory.removeSession(session.id);
+  let history = await db.noteHistory.get(id);
+  expect(history.length).toBe(0);
+});
+
+test("All sessions of a note should be cleared", async () => {
+  let { db, id } = await noteTest();
+  await sessionTest(db, id);
+
+  await db.noteHistory.clearSessions(id);
+
+  let history = await db.noteHistory.get(id);
+  expect(history.length).toBe(0);
+});
+
+test("Sessions should be serialized and deserialized", async () => {
+  let { db, id } = await noteTest();
+  await sessionTest(db, id);
+  let json = await db.noteHistory.serialize();
+  await db.noteHistory.clearSessions(id);
+  await db.noteHistory.deserialize(json);
+
+  let history = await db.noteHistory.get(id);
+  expect(history.length).toBe(1);
+  let content = await db.noteHistory.content(history[0].id);
+  expect(content).toBeTruthy();
+});
+
+test("String should compress and decompress", () => {
+  let compressed = compress(TEST_NOTE.content.data);
+  let decompressed = decompress(compressed);
+  expect(decompressed).toBe(TEST_NOTE.content.data);
+});

packages/core/api/index.js

@@ -23,6 +23,7 @@ import Offers from "./offers";
 import Attachments from "../collections/attachments";
 import Debug from "./debug";
 import { Mutex } from "async-mutex";
+import NoteHistory from "../collections/note-history";
 
 /**
  * @type {EventSource}
@@ -109,6 +110,8 @@ class Database {
     this.content = await Content.new(this, "content", false);
     /** @type {Attachments} */
     this.attachments = await Attachments.new(this, "attachments");
+    /**@type {NoteHistory} */
+    this.noteHistory = await NoteHistory.new(this,"notehistory");
 
     this.trash = new Trash(this);
 

packages/core/collections/contenthistory.js

@@ -0,0 +1,54 @@
+import { compress, decompress } from "../utils/compression";
+import { makeSessionContentId } from "../utils/id";
+import Collection from "./collection";
+
+export default class ContentHistory extends Collection {
+  /**
+   *
+   * @param {string} sessionId
+   * @param {{content:string:data:string}} content
+   */
+  async add(sessionId, content, locked) {
+    if (!sessionId || !content) return;
+    let compressed = locked ? null : compress(content.data);
+
+    await this._collection.addItem({
+      id: makeSessionContentId(sessionId),
+      data: compressed || content.data,
+      type: content.type,
+      compressed: !!compressed,
+      locked
+    });
+  }
+
+  /**
+   *
+   * @param {string} sessionId
+   * @returns {Promise<{content:string;data:string}>}
+   */
+  async get(sessionContentId) {
+    if (!sessionContentId) return;
+    let session = await this._collection.getItem(sessionContentId);
+    return {
+      data: session.compressed ? decompress(session.data) : session.data,
+      type: session.type,
+    };
+  }
+
+  /**
+   *
+   * @param {string} sessionContentId
+   */
+  async remove(sessionContentId) {
+    await this._collection.deleteItem(sessionContentId);
+  }
+
+
+  async all() {
+    let indices = await this._collection.indexer.getIndices();
+    let items =
+      (await this._collection.getItems(indices));
+
+    return Object.values(items);
+  }
+}

packages/core/collections/note-history.js

@@ -0,0 +1,213 @@
+import { makeSessionContentId, makeSessionId } from "../utils/id";
+import Collection from "./collection";
+import ContentHistory from "./contenthistory";
+/**
+ * @typedef Session
+ * @property {string} id
+ * @property {string} noteId
+ * @property {string} sessionContentId
+ * @property {string} dateEdited
+ * @property {string} dateCreated
+ * @property {boolean} locked
+ */
+
+/**
+ * @typedef Content
+ * @property {string} data
+ * @property {string} type
+ */
+
+export default class NoteHistory extends Collection {
+  constructor(db, name, cached) {
+    super(db, name, cached);
+    this.versionsLimit = 100;
+  }
+
+  async init() {
+    super.init();
+
+    /**
+     * @type {ContentHistory}
+     */
+    this.contentHistory = await ContentHistory.new(
+      this._db,
+      "contenthistory",
+      false
+    );
+  }
+
+  /**
+   * Get complete session history of a note.
+   * @param noteId id of the note
+   * @returns {Promise<Session[]>} An array of session ids of a note
+   */
+  async get(noteId) {
+    if (!noteId) return [];
+
+    let indices = await this._collection.indexer.getIndices();
+    let sessionIds = indices.filter((id) => id.includes(noteId));
+    if (sessionIds.length === 0) return [];
+    let history = (await this._collection.getItems(sessionIds)) || [];
+
+    return history;
+  }
+
+  /**
+   * Add and update a session content
+   * @param {string} noteId id of the note
+   * @param {string} dateEdited edited date of the note
+   * @param {Content} content
+   *
+   * @returns {Promise<Session>}
+   */
+  async add(noteId, dateEdited, content) {
+    if (!noteId || !dateEdited || !content) return;
+    let sessionId = `${noteId}_${dateEdited}`;
+    let exists = await this._collection.exists(sessionId);
+    let locked = this._db.notes.note(noteId)?.data?.locked;
+
+    let session = {
+      id: sessionId,
+      sessionContentId: makeSessionContentId(sessionId),
+      noteId,
+    };
+    if (!exists) {
+      session.dateCreated = Date.now();
+    }
+
+    if (locked) {
+      session.locked = true;
+    }
+
+    await this._collection.addItem(session);
+    await this.contentHistory.add(sessionId, content, session.locked);
+    await this._cleanup(noteId);
+
+    return session;
+  }
+
+  async _cleanup(noteId, limit = this.versionsLimit) {
+    let history = await this.get(noteId);
+    if (history.length === 0 || history.length < limit) return;
+    history.sort(function (a, b) {
+      return a.dateEdited - b.dateEdited;
+    });
+    let deleteCount = history.length - limit;
+
+    for (let i = 0; i < deleteCount; i++) {
+      let session = history[i];
+      await this._remove(session);
+    }
+  }
+
+  /**
+   * Get content of a session
+   * @param {string} sessionId session id
+   *
+   * @returns {Promise<Content>}
+   */
+  async content(sessionId) {
+    if (!sessionId) return;
+    return await this.contentHistory.get(sessionId);
+  }
+
+  /**
+   * Remove a session from storage
+   * @param {string} sessionId
+   */
+  async removeSession(sessionId) {
+    if (!sessionId) return;
+    /**
+     * @type {Session}
+     */
+    let session = this._collection.getItem(sessionId);
+    await this._remove(session);
+  }
+
+  /**
+   * Remove all sessions of a note from storage
+   * @param {string} noteId
+   */
+  async clearSessions(noteId) {
+    if (!noteId) return;
+    let history = await this.get(noteId);
+    for (let item of history) {
+      await this._remove(item);
+    }
+  }
+
+  /**
+   *
+   * @param {Session} session
+   */
+  async _remove(session) {
+    await this._collection.deleteItem(session.id);
+    await this.contentHistory.remove(session.sessionContentId);
+  }
+
+  /**
+   *
+   * @param {string} sessionId
+   */
+  async restore(sessionId) {
+    /**
+     * @type {Session}
+     */
+    let session = await this._collection.getItem(sessionId);
+    let content = await this.contentHistory.get(session.sessionContentId);
+    let note = this._db.notes.note(session.noteId).data;
+    if (session.locked) {
+      await this._db.content.add({
+        id: note.contentId,
+        data: content.data,
+        type: content.type,
+      });
+    } else {
+      await this._db.notes.add({
+        id: session.noteId,
+        content: {
+          data: content.data,
+          type: content.type,
+        },
+      });
+    }
+  }
+
+  /**
+   *
+   * @returns A json string containing all sessions with content
+   */
+  async serialize() {
+    return JSON.stringify({
+      sessions: await this.all(),
+      sessionContents: await this.contentHistory.all(),
+    });
+  }
+
+  async all() {
+    let indices = await this._collection.indexer.getIndices();
+    let items = await this._collection.getItems(indices);
+    return Object.values(items);
+  }
+
+  /**
+   * Restore session history from a serialized json string.
+   * @param {string} data
+   * @returns
+   */
+  async deserialize(data) {
+    if (!data) return;
+    let deserialized = JSON.parse(data);
+    if (!deserialized.sessions || !deserialized.sessionContents) return;
+
+    for (let session of deserialized.sessions) {
+      let sessionContent = deserialized.sessionContents.find((v) =>
+        v.id.includes(session.id)
+      );
+      if (sessionContent) {
+        await this._collection.addItem(session);
+        await this.contentHistory._collection.addItem(sessionContent);
+      }
+    }
+  }
+}

packages/core/package.json

@@ -25,9 +25,11 @@
   "dependencies": {
     "@stablelib/blake2s": "^1.0.1",
     "async-mutex": "^0.3.2",
+    "base64-arraybuffer": "^1.0.1",
     "dayjs": "^1.10.6",
     "fast-sort": "^2.0.1",
     "liqe": "^1.13.0",
+    "fflate": "^0.7.1",
     "node-html-parser": "github:thecodrr/node-html-parser",
     "qclone": "^1.0.4",
     "showdown": "github:thecodrr/showdown",

packages/core/utils/compression.js

@@ -0,0 +1,27 @@
+import { decode, encode } from "base64-arraybuffer";
+import fflate from "fflate";
+
+
+/**
+ * 
+ * @param {string} data
+ * @returns {string | null} An object containing compressed data
+ */
+export const compress = (data) => {
+  try {
+    return encode(fflate.compressSync(fflate.strToU8(data)).buffer)
+  } catch (e) {
+    return null;
+  }
+};
+
+/**
+ * 
+ * @param {string} compressed 
+ * @returns {string} decompressed string
+ */
+export const decompress = (compressed) => {
+  return fflate.strFromU8(
+    fflate.decompressSync(new Uint8Array(decode(compressed)))
+  );
+};

packages/core/utils/id.js

@@ -14,3 +14,12 @@ export default function () {
 export function makeId(text) {
   return SparkMD5.hash(text);
 }
+
+/**
+ *
+ * @param {string} noteId id of a note
+ * @returns {string} An id with postfix of "_index"
+ */
+export function makeSessionContentId(sessionId) {
+  return sessionId + "_content";
+}