fs: get rid of chunks property in file metadata

packages/streamable-fs/index.ts

@@ -20,6 +20,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import FileHandle from "./src/filehandle";
 import { IFileStorage, IStreamableFS } from "./src/interfaces";
 import { File } from "./src/types";
+import { chunkPrefix } from "./src/utils";
 
 export class StreamableFS implements IStreamableFS {
   /**
@@ -37,17 +38,19 @@ export class StreamableFS implements IStreamableFS {
     const file: File = {
       filename,
       size,
-      type,
-      chunks: 0
+      type
     };
     await this.storage.setMetadata(filename, file);
-    return new FileHandle(this.storage, file);
+    return new FileHandle(this.storage, file, []);
   }
 
   async readFile(filename: string): Promise<FileHandle | undefined> {
     const file = await this.storage.getMetadata(filename);
     if (!file) return undefined;
-    return new FileHandle(this.storage, file);
+    const chunks = (await this.storage.listChunks(chunkPrefix(filename))).sort(
+      (a, b) => a.localeCompare(b, undefined, { numeric: true })
+    );
+    return new FileHandle(this.storage, file, chunks);
   }
 
   async exists(filename: string): Promise<boolean> {

packages/streamable-fs/src/filehandle.ts

@@ -20,12 +20,19 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import FileStreamSource from "./filestreamsource";
 import { IFileStorage } from "./interfaces";
 import { File } from "./types";
+import { chunkPrefix } from "./utils";
 
 export default class FileHandle {
-  constructor(private readonly storage: IFileStorage, readonly file: File) {}
+  constructor(
+    private readonly storage: IFileStorage,
+    readonly file: File,
+    readonly chunks: string[]
+  ) {}
 
   get readable() {
-    return new ReadableStream(new FileStreamSource(this.storage, this.file));
+    return new ReadableStream(
+      new FileStreamSource(this.storage, this.file, this.chunks)
+    );
   }
 
   get writeable() {
@@ -33,20 +40,22 @@ export default class FileHandle {
       write: async (chunk, controller) => {
         if (controller.signal.aborted) return;
 
-        await this.storage.writeChunk(
-          this.getChunkKey(this.file.chunks++),
-          chunk
-        );
-        await this.storage.setMetadata(this.file.filename, this.file);
+        const lastOffset = this.lastOffset();
+        await this.storage.writeChunk(this.getChunkKey(lastOffset + 1), chunk);
+        this.chunks.push(this.getChunkKey(lastOffset + 1));
       },
       abort: async () => {
-        for (let i = 0; i < this.file.chunks; ++i) {
-          await this.storage.deleteChunk(this.getChunkKey(i));
+        for (const chunk of this.chunks) {
+          await this.storage.deleteChunk(chunk);
         }
       }
     });
   }
 
+  async writeChunkAtOffset(offset: number, chunk: Uint8Array) {
+    await this.storage.writeChunk(this.getChunkKey(offset), chunk);
+  }
+
   async addAdditionalData<T>(key: string, value: T) {
     this.file.additionalData = this.file.additionalData || {};
     this.file.additionalData[key] = value;
@@ -54,14 +63,14 @@ export default class FileHandle {
   }
 
   async delete() {
-    for (let i = 0; i < this.file.chunks; ++i) {
-      await this.storage.deleteChunk(this.getChunkKey(i));
+    for (const chunk of this.chunks) {
+      await this.storage.deleteChunk(chunk);
     }
     await this.storage.deleteMetadata(this.file.filename);
   }
 
   private getChunkKey(offset: number): string {
-    return `${this.file.filename}-chunk-${offset}`;
+    return `${chunkPrefix(this.file.filename)}${offset}`;
   }
 
   async readChunk(offset: number): Promise<Uint8Array | null> {
@@ -71,9 +80,9 @@ export default class FileHandle {
 
   async readChunks(from: number, length: number): Promise<Blob> {
     const blobParts: BlobPart[] = [];
-    for (let i = from; i < Math.min(from + length, this.file.chunks); ++i) {
+    for (let i = from; i < from + length; ++i) {
       const array = await this.readChunk(i);
-      if (!array) continue;
+      if (!array) throw new Error(`No data found for chunk at offset ${i}.`);
       blobParts.push(array.buffer);
     }
     return new Blob(blobParts, { type: this.file.type });
@@ -81,8 +90,8 @@ export default class FileHandle {
 
   async toBlob() {
     const blobParts: BlobPart[] = [];
-    for (let i = 0; i < this.file.chunks; ++i) {
-      const array = await this.readChunk(i);
+    for (const chunk of this.chunks) {
+      const array = await this.storage.readChunk(chunk);
       if (!array) continue;
       blobParts.push(array.buffer);
     }
@@ -91,11 +100,23 @@ export default class FileHandle {
 
   async size() {
     let size = 0;
-    for (let i = 0; i < this.file.chunks; ++i) {
-      const array = await this.readChunk(i);
+    for (const chunk of this.chunks) {
+      const array = await this.storage.readChunk(chunk);
       if (!array) continue;
       size += array.length;
     }
     return size;
   }
+
+  async listChunks() {
+    return (
+      await this.storage.listChunks(chunkPrefix(this.file.filename))
+    ).sort((a, b) => a.localeCompare(b, undefined, { numeric: true }));
+  }
+
+  private lastOffset() {
+    const lastChunk = this.chunks.at(-1);
+    if (!lastChunk) return -1;
+    return parseInt(lastChunk.replace(chunkPrefix(this.file.filename), ""));
+  }
 }

packages/streamable-fs/src/filestreamsource.ts

@@ -21,30 +21,27 @@ import { File } from "./types";
 import { IFileStorage } from "./interfaces";
 
 export default class FileStreamSource {
-  private offset = 0;
+  private index = 0;
 
   constructor(
     private readonly storage: IFileStorage,
-    private readonly file: File
+    private readonly file: File,
+    private readonly chunks: string[]
   ) {}
 
   start() {}
 
   async pull(controller: ReadableStreamDefaultController<Uint8Array>) {
-    const data = await this.readChunk(this.offset++);
+    const data = await this.readChunk(this.index++);
 
     if (data) controller.enqueue(data);
 
-    const isFinalChunk = this.offset === this.file.chunks;
+    const isFinalChunk = this.index === this.chunks.length;
     if (isFinalChunk || !data) controller.close();
   }
 
-  private readChunk(offset: number) {
-    if (offset > this.file.chunks) return;
-    return this.storage.readChunk(this.getChunkKey(offset));
-  }
-
-  private getChunkKey(offset: number): string {
-    return `${this.file.filename}-chunk-${offset}`;
+  private readChunk(index: number) {
+    if (index > this.chunks.length) return;
+    return this.storage.readChunk(this.chunks[index]);
   }
 }

packages/streamable-fs/src/interfaces.ts

@@ -36,4 +36,5 @@ export interface IFileStorage {
   writeChunk(chunkName: string, data: Uint8Array): Promise<void>;
   deleteChunk(chunkName: string): Promise<void>;
   readChunk(chunkName: string): Promise<Uint8Array | undefined>;
+  listChunks(chunkPrefix: string): Promise<string[]>;
 }

packages/streamable-fs/src/types.ts

@@ -21,6 +21,6 @@ export type File = {
   filename: string;
   size: number;
   type: string;
-  chunks: number;
+  //  chunks: number;
   additionalData?: { [key: string]: unknown };
 };

packages/streamable-fs/src/utils.ts

@@ -0,0 +1,22 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+export function chunkPrefix(filename: string) {
+  return `${filename}-chunk-`;
+}