diff --git a/packages/streamable-fs/index.ts b/packages/streamable-fs/index.ts
index d2bc9b758..0a200d7bb 100644
--- a/packages/streamable-fs/index.ts
+++ b/packages/streamable-fs/index.ts
@@ -20,6 +20,7 @@ along with this program. If not, see .
import FileHandle from "./src/filehandle";
import { IFileStorage, IStreamableFS } from "./src/interfaces";
import { File } from "./src/types";
+import { chunkPrefix } from "./src/utils";
export class StreamableFS implements IStreamableFS {
/**
@@ -37,17 +38,19 @@ export class StreamableFS implements IStreamableFS {
const file: File = {
filename,
size,
- type,
- chunks: 0
+ type
};
await this.storage.setMetadata(filename, file);
- return new FileHandle(this.storage, file);
+ return new FileHandle(this.storage, file, []);
}
async readFile(filename: string): Promise {
const file = await this.storage.getMetadata(filename);
if (!file) return undefined;
- return new FileHandle(this.storage, file);
+ const chunks = (await this.storage.listChunks(chunkPrefix(filename))).sort(
+ (a, b) => a.localeCompare(b, undefined, { numeric: true })
+ );
+ return new FileHandle(this.storage, file, chunks);
}
async exists(filename: string): Promise {
diff --git a/packages/streamable-fs/src/filehandle.ts b/packages/streamable-fs/src/filehandle.ts
index 04f66c8bd..db1a20239 100644
--- a/packages/streamable-fs/src/filehandle.ts
+++ b/packages/streamable-fs/src/filehandle.ts
@@ -20,12 +20,19 @@ along with this program. If not, see .
import FileStreamSource from "./filestreamsource";
import { IFileStorage } from "./interfaces";
import { File } from "./types";
+import { chunkPrefix } from "./utils";
export default class FileHandle {
- constructor(private readonly storage: IFileStorage, readonly file: File) {}
+ constructor(
+ private readonly storage: IFileStorage,
+ readonly file: File,
+ readonly chunks: string[]
+ ) {}
get readable() {
- return new ReadableStream(new FileStreamSource(this.storage, this.file));
+ return new ReadableStream(
+ new FileStreamSource(this.storage, this.file, this.chunks)
+ );
}
get writeable() {
@@ -33,20 +40,22 @@ export default class FileHandle {
write: async (chunk, controller) => {
if (controller.signal.aborted) return;
- await this.storage.writeChunk(
- this.getChunkKey(this.file.chunks++),
- chunk
- );
- await this.storage.setMetadata(this.file.filename, this.file);
+ const lastOffset = this.lastOffset();
+ await this.storage.writeChunk(this.getChunkKey(lastOffset + 1), chunk);
+ this.chunks.push(this.getChunkKey(lastOffset + 1));
},
abort: async () => {
- for (let i = 0; i < this.file.chunks; ++i) {
- await this.storage.deleteChunk(this.getChunkKey(i));
+ for (const chunk of this.chunks) {
+ await this.storage.deleteChunk(chunk);
}
}
});
}
+ async writeChunkAtOffset(offset: number, chunk: Uint8Array) {
+ await this.storage.writeChunk(this.getChunkKey(offset), chunk);
+ }
+
async addAdditionalData(key: string, value: T) {
this.file.additionalData = this.file.additionalData || {};
this.file.additionalData[key] = value;
@@ -54,14 +63,14 @@ export default class FileHandle {
}
async delete() {
- for (let i = 0; i < this.file.chunks; ++i) {
- await this.storage.deleteChunk(this.getChunkKey(i));
+ for (const chunk of this.chunks) {
+ await this.storage.deleteChunk(chunk);
}
await this.storage.deleteMetadata(this.file.filename);
}
private getChunkKey(offset: number): string {
- return `${this.file.filename}-chunk-${offset}`;
+ return `${chunkPrefix(this.file.filename)}${offset}`;
}
async readChunk(offset: number): Promise {
@@ -71,9 +80,9 @@ export default class FileHandle {
async readChunks(from: number, length: number): Promise {
const blobParts: BlobPart[] = [];
- for (let i = from; i < Math.min(from + length, this.file.chunks); ++i) {
+ for (let i = from; i < from + length; ++i) {
const array = await this.readChunk(i);
- if (!array) continue;
+ if (!array) throw new Error(`No data found for chunk at offset ${i}.`);
blobParts.push(array.buffer);
}
return new Blob(blobParts, { type: this.file.type });
@@ -81,8 +90,8 @@ export default class FileHandle {
async toBlob() {
const blobParts: BlobPart[] = [];
- for (let i = 0; i < this.file.chunks; ++i) {
- const array = await this.readChunk(i);
+ for (const chunk of this.chunks) {
+ const array = await this.storage.readChunk(chunk);
if (!array) continue;
blobParts.push(array.buffer);
}
@@ -91,11 +100,23 @@ export default class FileHandle {
async size() {
let size = 0;
- for (let i = 0; i < this.file.chunks; ++i) {
- const array = await this.readChunk(i);
+ for (const chunk of this.chunks) {
+ const array = await this.storage.readChunk(chunk);
if (!array) continue;
size += array.length;
}
return size;
}
+
+ async listChunks() {
+ return (
+ await this.storage.listChunks(chunkPrefix(this.file.filename))
+ ).sort((a, b) => a.localeCompare(b, undefined, { numeric: true }));
+ }
+
+ private lastOffset() {
+ const lastChunk = this.chunks.at(-1);
+ if (!lastChunk) return -1;
+ return parseInt(lastChunk.replace(chunkPrefix(this.file.filename), ""));
+ }
}
diff --git a/packages/streamable-fs/src/filestreamsource.ts b/packages/streamable-fs/src/filestreamsource.ts
index 8a554466d..1b86d8ba3 100644
--- a/packages/streamable-fs/src/filestreamsource.ts
+++ b/packages/streamable-fs/src/filestreamsource.ts
@@ -21,30 +21,27 @@ import { File } from "./types";
import { IFileStorage } from "./interfaces";
export default class FileStreamSource {
- private offset = 0;
+ private index = 0;
constructor(
private readonly storage: IFileStorage,
- private readonly file: File
+ private readonly file: File,
+ private readonly chunks: string[]
) {}
start() {}
async pull(controller: ReadableStreamDefaultController) {
- const data = await this.readChunk(this.offset++);
+ const data = await this.readChunk(this.index++);
if (data) controller.enqueue(data);
- const isFinalChunk = this.offset === this.file.chunks;
+ const isFinalChunk = this.index === this.chunks.length;
if (isFinalChunk || !data) controller.close();
}
- private readChunk(offset: number) {
- if (offset > this.file.chunks) return;
- return this.storage.readChunk(this.getChunkKey(offset));
- }
-
- private getChunkKey(offset: number): string {
- return `${this.file.filename}-chunk-${offset}`;
+ private readChunk(index: number) {
+ if (index > this.chunks.length) return;
+ return this.storage.readChunk(this.chunks[index]);
}
}
diff --git a/packages/streamable-fs/src/interfaces.ts b/packages/streamable-fs/src/interfaces.ts
index a4f646c22..74af3382d 100644
--- a/packages/streamable-fs/src/interfaces.ts
+++ b/packages/streamable-fs/src/interfaces.ts
@@ -36,4 +36,5 @@ export interface IFileStorage {
writeChunk(chunkName: string, data: Uint8Array): Promise;
deleteChunk(chunkName: string): Promise;
readChunk(chunkName: string): Promise;
+ listChunks(chunkPrefix: string): Promise;
}
diff --git a/packages/streamable-fs/src/types.ts b/packages/streamable-fs/src/types.ts
index accbb60ed..453303779 100644
--- a/packages/streamable-fs/src/types.ts
+++ b/packages/streamable-fs/src/types.ts
@@ -21,6 +21,6 @@ export type File = {
filename: string;
size: number;
type: string;
- chunks: number;
+ // chunks: number;
additionalData?: { [key: string]: unknown };
};
diff --git a/packages/streamable-fs/src/utils.ts b/packages/streamable-fs/src/utils.ts
new file mode 100644
index 000000000..d53be8e67
--- /dev/null
+++ b/packages/streamable-fs/src/utils.ts
@@ -0,0 +1,22 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see .
+*/
+
+export function chunkPrefix(filename: string) {
+ return `${filename}-chunk-`;
+}