core: migrate attachments to use relations for linking to notes

packages/core/__tests__/backup.test.js

@@ -174,6 +174,8 @@ describe.each([
         db.attachments.all.every((v) => v.dateModified > 0 && !v.dateEdited)
       ).toBeTruthy();
 
+      expect(db.attachments.all.every((a) => !a.noteIds)).toBeTruthy();
+
       if (data.data.settings.pins)
         expect(db.shortcuts.all).toHaveLength(data.data.settings.pins.length);
 

packages/core/src/api/sync/merger.ts

@@ -17,7 +17,6 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-import { set } from "../../utils/set";
 import { logger } from "../../logger";
 import { isHTMLEqual } from "../../utils/html-diff";
 import Database from "..";
@@ -201,9 +200,8 @@ class Merger {
         break;
       }
       case "attachment": {
-        if (isDeleted(remoteItem)) {
-          return this.db.attachments.merge(undefined, remoteItem);
-        }
+        if (isDeleted(remoteItem)) return remoteItem;
+
         if (remoteItem.type !== "attachment") return;
 
         const localAttachment = this.db.attachments.attachment(
@@ -213,7 +211,6 @@ class Merger {
           localAttachment &&
           localAttachment.dateUploaded !== remoteItem.dateUploaded
         ) {
-          const noteIds = localAttachment.noteIds.slice();
           const isRemoved = await this.db.attachments.remove(
             localAttachment.metadata.hash,
             true
@@ -222,9 +219,8 @@ class Merger {
             throw new Error(
               "Conflict could not be resolved in one of the attachments."
             );
-          remoteItem.noteIds = set.union(remoteItem.noteIds, noteIds);
         }
-        return this.db.attachments.merge(undefined, remoteItem);
+        return remoteItem;
       }
     }
   }

packages/core/src/collections/attachments.ts

@@ -19,11 +19,10 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import { ICollection } from "./collection";
 import { getId } from "../utils/id";
-import { deleteItem, hasItem } from "../utils/array";
+import { hasItem } from "../utils/array";
 import { EV, EVENTS } from "../common";
 import dataurl from "../utils/dataurl";
 import dayjs from "dayjs";
-import { set } from "../utils/set";
 import {
   getFileNameWithExtension,
   isImage,
@@ -32,12 +31,7 @@ import {
 import { Cipher, DataFormat, SerializedKey } from "@notesnook/crypto";
 import { CachedCollection } from "../database/cached-collection";
 import { Output } from "../interfaces";
-import {
-  Attachment,
-  AttachmentMetadata,
-  MaybeDeletedItem,
-  isDeleted
-} from "../types";
+import { Attachment, AttachmentMetadata, isDeleted } from "../types";
 import Database from "../api";
 
 export class Attachments implements ICollection {
@@ -108,27 +102,6 @@ export class Attachments implements ICollection {
     await this.collection.init();
   }
 
-  merge(
-    localAttachment: MaybeDeletedItem<Attachment> | undefined,
-    remoteAttachment: MaybeDeletedItem<Attachment>
-  ) {
-    if (isDeleted(remoteAttachment)) return remoteAttachment;
-
-    if (
-      localAttachment &&
-      !isDeleted(localAttachment) &&
-      localAttachment.noteIds
-    ) {
-      remoteAttachment.noteIds = set.union(
-        remoteAttachment.noteIds,
-        localAttachment.noteIds
-      );
-      remoteAttachment.remote = false;
-    }
-
-    return remoteAttachment;
-  }
-
   async add(
     item: Partial<
       Omit<Attachment, "key" | "metadata"> & {
@@ -145,7 +118,6 @@ export class Attachments implements ICollection {
       (a) => a.metadata.hash === item.metadata?.hash
     );
     const id = oldAttachment?.id || getId();
-    const noteIds = set.union(oldAttachment?.noteIds || [], item.noteIds || []);
 
     const encryptedKey = item.key
       ? await this.encryptKey(item.key)
@@ -154,7 +126,6 @@ export class Attachments implements ICollection {
       ...oldAttachment,
       ...oldAttachment?.metadata,
       ...item,
-      noteIds,
       key: encryptedKey
     };
 
@@ -193,7 +164,6 @@ export class Attachments implements ICollection {
     return this.collection.add({
       type: "attachment",
       id,
-      noteIds,
       iv,
       salt,
       length,
@@ -226,21 +196,11 @@ export class Attachments implements ICollection {
     return JSON.parse(plainData);
   }
 
-  async delete(hashOrId: string, noteId: string) {
-    const attachment = this.attachment(hashOrId);
-    if (!attachment || !deleteItem(attachment.noteIds, noteId)) return;
-    if (!attachment.noteIds.length) {
-      attachment.dateDeleted = Date.now();
-      EV.publish(EVENTS.attachmentDeleted, attachment);
-    }
-    return await this.collection.update(attachment);
-  }
-
   async remove(hashOrId: string, localOnly: boolean) {
     const attachment = this.attachment(hashOrId);
     if (!attachment || !attachment.metadata) return false;
 
-    if (!localOnly && !(await this._canDetach(attachment)))
+    if (!localOnly && !(await this.canDetach(attachment)))
       throw new Error("This attachment is inside a locked note.");
 
     if (
@@ -261,38 +221,37 @@ export class Attachments implements ICollection {
   }
 
   async detach(attachment: Attachment) {
-    for (const noteId of attachment.noteIds) {
-      const note = this.db.notes.note(noteId);
-      if (!note || !note.data.contentId) continue;
-      await this.db.content.removeAttachments(note.data.contentId, [
+    for (const note of this.db.relations.from(attachment, "note").resolved()) {
+      if (!note || !note.contentId) continue;
+      await this.db.content.removeAttachments(note.contentId, [
         attachment.metadata.hash
       ]);
     }
   }
 
-  async _canDetach(attachment: Attachment) {
-    for (const noteId of attachment.noteIds) {
-      const note = this.db.notes?.note(noteId);
-      if (note && note.data.locked) return false;
-    }
-
-    return true;
+  private async canDetach(attachment: Attachment) {
+    return this.db.relations
+      .from(attachment, "note")
+      .resolved()
+      .every((note) => !note.locked);
   }
 
   ofNote(
     noteId: string,
-    type: "files" | "images" | "webclips" | "all"
+    ...types: ("files" | "images" | "webclips" | "all")[]
   ): Attachment[] {
-    let attachments: Attachment[] = [];
+    const noteAttachments = this.db.relations
+      .from({ type: "note", id: noteId }, "attachment")
+      .resolved();
 
-    if (type === "files") attachments = this.files;
-    else if (type === "images") attachments = this.images;
-    else if (type === "webclips") attachments = this.webclips;
-    else if (type === "all") attachments = this.all;
+    if (types.includes("all")) return noteAttachments;
 
-    return attachments.filter((attachment) =>
-      hasItem(attachment.noteIds, noteId)
-    );
+    return noteAttachments.filter((a) => {
+      if (isImage(a.metadata.type) && types.includes("images")) return true;
+      else if (isWebClip(a.metadata.type) && types.includes("webclips"))
+        return true;
+      else if (types.includes("files")) return true;
+    });
   }
 
   exists(hash: string) {
@@ -387,12 +346,11 @@ export class Attachments implements ICollection {
   }
 
   async downloadMedia(noteId: string, hashesToLoad?: string[]) {
-    const attachments = this.media.filter(
-      (attachment) =>
-        !!attachment.metadata &&
-        hasItem(attachment.noteIds, noteId) &&
-        (!hashesToLoad || hasItem(hashesToLoad, attachment.metadata?.hash))
-    );
+    let attachments = this.ofNote(noteId, "images", "webclips");
+    if (hashesToLoad)
+      attachments = attachments.filter((a) =>
+        hasItem(hashesToLoad, a.metadata.hash)
+      );
 
     await this.db.fs().queueDownloads(
       attachments.map((a) => ({

packages/core/src/collections/content.ts

@@ -20,7 +20,6 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import { ICollection } from "./collection";
 import { getId } from "../utils/id";
 import { getContentFromData } from "../content-types";
-import { hasItem } from "../utils/array";
 import { ResolveHashes } from "../content-types/tiptap";
 import { isCipher } from "../database/crypto";
 import {
@@ -238,16 +237,15 @@ export class Content implements ICollection {
   async extractAttachments(contentItem: UnencryptedContentItem) {
     if (contentItem.localOnly) return contentItem;
 
-    const allAttachments = this.db.attachments?.all;
     const content = getContentFromData(contentItem.type, contentItem.data);
     if (!content) return contentItem;
     const { data, hashes } = await content.extractAttachments(
       this.db.attachments.save
     );
 
-    const noteAttachments = allAttachments.filter((attachment) =>
-      hasItem(attachment.noteIds, contentItem.noteId)
-    );
+    const noteAttachments = this.db.relations
+      .from({ type: "note", id: contentItem.noteId }, "attachment")
+      .resolved();
 
     const toDelete = noteAttachments.filter((attachment) => {
       return hashes.every((hash) => hash !== attachment.metadata.hash);
@@ -258,17 +256,25 @@ export class Content implements ICollection {
     });
 
     for (const attachment of toDelete) {
-      await this.db.attachments.delete(
-        attachment.metadata.hash,
-        contentItem.noteId
+      await this.db.relations.unlink(
+        {
+          id: contentItem.noteId,
+          type: "note"
+        },
+        attachment
       );
     }
 
     for (const hash of toAdd) {
-      await this.db.attachments.add({
-        noteIds: [contentItem.noteId],
-        metadata: { hash }
-      });
+      const attachment = this.db.attachments.attachment(hash);
+      if (!attachment) continue;
+      await this.db.relations.add(
+        {
+          id: contentItem.noteId,
+          type: "note"
+        },
+        attachment
+      );
     }
 
     if (toAdd.length > 0) {

packages/core/src/collections/notes.ts

@@ -302,11 +302,7 @@ export class Notes implements ICollection {
 
       await this.db.relations.unlinkAll(item, "tag");
       await this.db.relations.unlinkAll(item, "color");
-
-      const attachments = this.db.attachments.ofNote(itemData.id, "all");
-      for (const attachment of attachments) {
-        await this.db.attachments.delete(attachment.metadata.hash, itemData.id);
-      }
+      await this.db.relations.unlinkAll(item, "attachment");
 
       if (moveToTrash && !isTrashItem(itemData))
         await this.db.trash.add(itemData);

packages/core/src/collections/relations.ts

@@ -168,6 +168,12 @@ export class Relations implements ICollection {
           item = notebook.data;
           break;
         }
+        case "attachment": {
+          const attachment = this.db.attachments.attachment(reference.id);
+          if (!attachment) continue;
+          item = attachment;
+          break;
+        }
       }
       if (item) items.push(item);
     }

packages/core/src/database/cached-collection.ts

@@ -20,9 +20,9 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import { IndexedCollection } from "./indexed-collection";
 import MapStub from "../utils/map";
 import {
-  BaseItem,
   CollectionType,
   Collections,
+  ItemMap,
   MaybeDeletedItem,
   isDeleted
 } from "../types";
@@ -32,7 +32,7 @@ import { chunkedIterate } from "../utils/array";
 
 export class CachedCollection<
   TCollectionType extends CollectionType,
-  T extends BaseItem<Collections[TCollectionType]>
+  T extends ItemMap[Collections[TCollectionType]]
 > {
   private collection: IndexedCollection<TCollectionType, T>;
   private cache = new Map<string, MaybeDeletedItem<T>>();

packages/core/src/migrations.ts

@@ -249,6 +249,16 @@ const migrations: Migration[] = [
         delete item.tags;
         delete item.color;
         return true;
+      },
+      attachment: async (item, db) => {
+        for (const noteId of item.noteIds || []) {
+          await db.relations.add(
+            { type: "attachment", id: item.id },
+            { type: "note", id: noteId }
+          );
+        }
+        delete item.noteIds;
+        return true;
       }
     }
   },

packages/core/src/types.ts

@@ -178,7 +178,6 @@ export type AttachmentMetadata = {
 };
 
 export interface Attachment extends BaseItem<"attachment"> {
-  noteIds: string[];
   iv: string;
   salt: string;
   length: number;
@@ -189,6 +188,11 @@ export interface Attachment extends BaseItem<"attachment"> {
   dateUploaded?: number;
   failed?: string;
   dateDeleted?: number;
+
+  /**
+   * @deprecated only kept here for migration purposes
+   */
+  noteIds?: string[];
 }
 
 export interface Color extends BaseItem<"color"> {