From 053f70cdb44f6a85bc94758cc9d451ab2665c368 Mon Sep 17 00:00:00 2001 From: Abdullah Atta Date: Wed, 24 Jul 2024 12:35:41 +0500 Subject: [PATCH] fs: get rid of chunks property in file metadata --- packages/streamable-fs/index.ts | 11 ++-- packages/streamable-fs/src/filehandle.ts | 57 +++++++++++++------ .../streamable-fs/src/filestreamsource.ts | 19 +++---- packages/streamable-fs/src/interfaces.ts | 1 + packages/streamable-fs/src/types.ts | 2 +- packages/streamable-fs/src/utils.ts | 22 +++++++ 6 files changed, 78 insertions(+), 34 deletions(-) create mode 100644 packages/streamable-fs/src/utils.ts diff --git a/packages/streamable-fs/index.ts b/packages/streamable-fs/index.ts index d2bc9b758..0a200d7bb 100644 --- a/packages/streamable-fs/index.ts +++ b/packages/streamable-fs/index.ts @@ -20,6 +20,7 @@ along with this program. If not, see . import FileHandle from "./src/filehandle"; import { IFileStorage, IStreamableFS } from "./src/interfaces"; import { File } from "./src/types"; +import { chunkPrefix } from "./src/utils"; export class StreamableFS implements IStreamableFS { /** @@ -37,17 +38,19 @@ export class StreamableFS implements IStreamableFS { const file: File = { filename, size, - type, - chunks: 0 + type }; await this.storage.setMetadata(filename, file); - return new FileHandle(this.storage, file); + return new FileHandle(this.storage, file, []); } async readFile(filename: string): Promise { const file = await this.storage.getMetadata(filename); if (!file) return undefined; - return new FileHandle(this.storage, file); + const chunks = (await this.storage.listChunks(chunkPrefix(filename))).sort( + (a, b) => a.localeCompare(b, undefined, { numeric: true }) + ); + return new FileHandle(this.storage, file, chunks); } async exists(filename: string): Promise { diff --git a/packages/streamable-fs/src/filehandle.ts b/packages/streamable-fs/src/filehandle.ts index 04f66c8bd..db1a20239 100644 --- a/packages/streamable-fs/src/filehandle.ts +++ b/packages/streamable-fs/src/filehandle.ts @@ -20,12 +20,19 @@ along with this program. If not, see . import FileStreamSource from "./filestreamsource"; import { IFileStorage } from "./interfaces"; import { File } from "./types"; +import { chunkPrefix } from "./utils"; export default class FileHandle { - constructor(private readonly storage: IFileStorage, readonly file: File) {} + constructor( + private readonly storage: IFileStorage, + readonly file: File, + readonly chunks: string[] + ) {} get readable() { - return new ReadableStream(new FileStreamSource(this.storage, this.file)); + return new ReadableStream( + new FileStreamSource(this.storage, this.file, this.chunks) + ); } get writeable() { @@ -33,20 +40,22 @@ export default class FileHandle { write: async (chunk, controller) => { if (controller.signal.aborted) return; - await this.storage.writeChunk( - this.getChunkKey(this.file.chunks++), - chunk - ); - await this.storage.setMetadata(this.file.filename, this.file); + const lastOffset = this.lastOffset(); + await this.storage.writeChunk(this.getChunkKey(lastOffset + 1), chunk); + this.chunks.push(this.getChunkKey(lastOffset + 1)); }, abort: async () => { - for (let i = 0; i < this.file.chunks; ++i) { - await this.storage.deleteChunk(this.getChunkKey(i)); + for (const chunk of this.chunks) { + await this.storage.deleteChunk(chunk); } } }); } + async writeChunkAtOffset(offset: number, chunk: Uint8Array) { + await this.storage.writeChunk(this.getChunkKey(offset), chunk); + } + async addAdditionalData(key: string, value: T) { this.file.additionalData = this.file.additionalData || {}; this.file.additionalData[key] = value; @@ -54,14 +63,14 @@ export default class FileHandle { } async delete() { - for (let i = 0; i < this.file.chunks; ++i) { - await this.storage.deleteChunk(this.getChunkKey(i)); + for (const chunk of this.chunks) { + await this.storage.deleteChunk(chunk); } await this.storage.deleteMetadata(this.file.filename); } private getChunkKey(offset: number): string { - return `${this.file.filename}-chunk-${offset}`; + return `${chunkPrefix(this.file.filename)}${offset}`; } async readChunk(offset: number): Promise { @@ -71,9 +80,9 @@ export default class FileHandle { async readChunks(from: number, length: number): Promise { const blobParts: BlobPart[] = []; - for (let i = from; i < Math.min(from + length, this.file.chunks); ++i) { + for (let i = from; i < from + length; ++i) { const array = await this.readChunk(i); - if (!array) continue; + if (!array) throw new Error(`No data found for chunk at offset ${i}.`); blobParts.push(array.buffer); } return new Blob(blobParts, { type: this.file.type }); @@ -81,8 +90,8 @@ export default class FileHandle { async toBlob() { const blobParts: BlobPart[] = []; - for (let i = 0; i < this.file.chunks; ++i) { - const array = await this.readChunk(i); + for (const chunk of this.chunks) { + const array = await this.storage.readChunk(chunk); if (!array) continue; blobParts.push(array.buffer); } @@ -91,11 +100,23 @@ export default class FileHandle { async size() { let size = 0; - for (let i = 0; i < this.file.chunks; ++i) { - const array = await this.readChunk(i); + for (const chunk of this.chunks) { + const array = await this.storage.readChunk(chunk); if (!array) continue; size += array.length; } return size; } + + async listChunks() { + return ( + await this.storage.listChunks(chunkPrefix(this.file.filename)) + ).sort((a, b) => a.localeCompare(b, undefined, { numeric: true })); + } + + private lastOffset() { + const lastChunk = this.chunks.at(-1); + if (!lastChunk) return -1; + return parseInt(lastChunk.replace(chunkPrefix(this.file.filename), "")); + } } diff --git a/packages/streamable-fs/src/filestreamsource.ts b/packages/streamable-fs/src/filestreamsource.ts index 8a554466d..1b86d8ba3 100644 --- a/packages/streamable-fs/src/filestreamsource.ts +++ b/packages/streamable-fs/src/filestreamsource.ts @@ -21,30 +21,27 @@ import { File } from "./types"; import { IFileStorage } from "./interfaces"; export default class FileStreamSource { - private offset = 0; + private index = 0; constructor( private readonly storage: IFileStorage, - private readonly file: File + private readonly file: File, + private readonly chunks: string[] ) {} start() {} async pull(controller: ReadableStreamDefaultController) { - const data = await this.readChunk(this.offset++); + const data = await this.readChunk(this.index++); if (data) controller.enqueue(data); - const isFinalChunk = this.offset === this.file.chunks; + const isFinalChunk = this.index === this.chunks.length; if (isFinalChunk || !data) controller.close(); } - private readChunk(offset: number) { - if (offset > this.file.chunks) return; - return this.storage.readChunk(this.getChunkKey(offset)); - } - - private getChunkKey(offset: number): string { - return `${this.file.filename}-chunk-${offset}`; + private readChunk(index: number) { + if (index > this.chunks.length) return; + return this.storage.readChunk(this.chunks[index]); } } diff --git a/packages/streamable-fs/src/interfaces.ts b/packages/streamable-fs/src/interfaces.ts index a4f646c22..74af3382d 100644 --- a/packages/streamable-fs/src/interfaces.ts +++ b/packages/streamable-fs/src/interfaces.ts @@ -36,4 +36,5 @@ export interface IFileStorage { writeChunk(chunkName: string, data: Uint8Array): Promise; deleteChunk(chunkName: string): Promise; readChunk(chunkName: string): Promise; + listChunks(chunkPrefix: string): Promise; } diff --git a/packages/streamable-fs/src/types.ts b/packages/streamable-fs/src/types.ts index accbb60ed..453303779 100644 --- a/packages/streamable-fs/src/types.ts +++ b/packages/streamable-fs/src/types.ts @@ -21,6 +21,6 @@ export type File = { filename: string; size: number; type: string; - chunks: number; + // chunks: number; additionalData?: { [key: string]: unknown }; }; diff --git a/packages/streamable-fs/src/utils.ts b/packages/streamable-fs/src/utils.ts new file mode 100644 index 000000000..d53be8e67 --- /dev/null +++ b/packages/streamable-fs/src/utils.ts @@ -0,0 +1,22 @@ +/* +This file is part of the Notesnook project (https://notesnook.com/) + +Copyright (C) 2023 Streetwriters (Private) Limited + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +export function chunkPrefix(filename: string) { + return `${filename}-chunk-`; +}