Mirrors/notesnook
1
0
Fork 0
mirror of https://github.com/streetwriters/notesnook.git synced 2026-02-23 19:49:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

web: add zip64 support for backups, export & imports (#4079)

Browse Source 
* web: use @zip.js/zip.js for reading zip files

* web: always use zip streams for writing zips

* core: fix types

* theme: update lockfile
This commit is contained in:
Abdullah Atta
2024-01-05 22:05:04 +05:00
committed by GitHub
parent 248fb6dd2d
commit 674881b873
17 changed files with 3123 additions and 736 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2820
apps/web/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

3
apps/web/package.json
View File

@@ -16,7 +16,7 @@
"@emotion/react": "11.11.1",
"@mdi/js": "^7.2.96",
"@mdi/react": "^1.6.1",
"@notesnook-importer/core": "^1.7.1",
"@notesnook-importer/core": "^2.0.0",
"@notesnook/common": "file:../../packages/common",
"@notesnook/core": "file:../../packages/core",
"@notesnook/crypto": "file:../../packages/crypto",
@@ -36,6 +36,7 @@
"@theme-ui/core": "^0.14.7",
"@trpc/client": "10.38.3",
"@trpc/react-query": "10.38.3",
"@zip.js/zip.js": "^2.7.32",
"allotment": "^1.19.0",
"axios": "^1.3.4",
"clipboard-polyfill": "4.0.0",

91
apps/web/src/common/export.ts
View File

@@ -17,21 +17,10 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { db } from "./db";
import { TaskManager } from "./task-manager";
import { zip } from "../utils/zip";
import { saveAs } from "file-saver";
import { showToast } from "../utils/toast";
import { sanitizeFilename } from "@notesnook/common";
import Vault from "./vault";
const FORMAT_TO_EXT = {
pdf: "pdf",
md: "md",
txt: "txt",
html: "html",
"md-frontmatter": "md"
} as const;
import { ExportStream } from "../utils/streams/export-stream";
import { createZipStream } from "../utils/streams/zip-stream";
import { createWriteStream } from "../utils/stream-saver";
export async function exportToPDF(
title: string,
@@ -85,75 +74,11 @@ export async function exportNotes(
title: "Exporting notes",
subtitle: "Please wait while your notes are exported.",
action: async (report) => {
let vaultUnlocked = false;
if (noteIds.length === 1 && db.notes?.note(noteIds[0])?.data.locked) {
vaultUnlocked = await Vault.unlockVault();
if (!vaultUnlocked) return false;
} else if (noteIds.length > 1 && (await db.vault?.exists())) {
vaultUnlocked = await Vault.unlockVault();
if (!vaultUnlocked)
showToast(
"error",
"Failed to unlock vault. Locked notes will be skipped."
);
}
const files = [];
let index = 0;
for (const noteId of noteIds) {
const note = db.notes?.note(noteId);
if (!note) continue;
if (!vaultUnlocked && note.data.locked) continue;
report({
current: ++index,
total: noteIds.length,
text: `Exporting "${note.title}"...`
});
const rawContent = await db.content?.raw(note.data.contentId);
const content = note.data.locked
? await db.vault?.decryptContent(rawContent)
: rawContent;
const exported = await note
.export(format === "pdf" ? "html" : format, content)
.catch((e: Error) => {
console.error(note.data, e);
showToast(
"error",
`Failed to export note "${note.title}": ${e.message}`
);
});
if (typeof exported !== "string") {
showToast("error", `Failed to export note "${note.title}"`);
continue;
}
if (format === "pdf") {
return await exportToPDF(note.title, exported);
}
files.push({
filename: note.title,
content: exported,
date: note.dateEdited
});
}
if (!files.length) return false;
if (files.length === 1) {
saveAs(
new Blob([Buffer.from(files[0].content, "utf-8")]),
`${sanitizeFilename(files[0].filename)}.${FORMAT_TO_EXT[format]}`
);
} else {
const zipped = await zip(files, FORMAT_TO_EXT[format]);
saveAs(new Blob([zipped.buffer]), "notes.zip");
}
await new ExportStream(noteIds, format, undefined, (c, text) =>
report({ total: noteIds.length, current: c, text })
)
.pipeThrough(createZipStream())
.pipeTo(await createWriteStream("notes.zip"));
return true;
}
});

13
apps/web/src/common/index.ts
View File

@@ -36,9 +36,9 @@ import { TaskManager } from "./task-manager";
import { EVENTS } from "@notesnook/core/dist/common";
import { getFormattedDate } from "@notesnook/common";
import { createWritableStream } from "./desktop-bridge";
import { ZipStream } from "../utils/streams/zip-stream";
import { createZipStream } from "../utils/streams/zip-stream";
import { FeatureKeys } from "../dialogs/feature-dialog";
import { Entry, Reader } from "../utils/zip-reader";
import { ZipEntry, createUnzipIterator } from "../utils/streams/unzip-stream";
export const CREATE_BUTTON_MAP = {
notes: {
@@ -113,7 +113,7 @@ export async function createBackup() {
controller.close();
}
})
.pipeThrough(new ZipStream())
.pipeThrough(createZipStream())
.pipeTo(writeStream);
}
});
@@ -166,12 +166,12 @@ export async function restoreBackupFile(backupFile: File) {
type: "modal",
action: async (report) => {
let cachedPassword: string | undefined = undefined;
const { read, totalFiles } = await Reader(backupFile);
const entries: Entry[] = [];
// const { read, totalFiles } = await Reader(backupFile);
const entries: ZipEntry[] = [];
let filesProcessed = 0;
let isValid = false;
for await (const entry of read()) {
for await (const entry of createUnzipIterator(backupFile)) {
if (entry.name === ".nnbackup") {
isValid = true;
continue;
@@ -200,7 +200,6 @@ export async function restoreBackupFile(backupFile: File) {
}
report({
total: totalFiles,
text: `Processed ${entry.name}`,
current: filesProcessed++
});

6
apps/web/src/dialogs/settings/components/importer.tsx
View File

@@ -149,13 +149,9 @@ export function Importer() {
setErrors((errors) => [...errors, message.error]);
break;
case "progress": {
const { count, filesRead, totalFiles } = message;
const { count } = message;
if (notesCounter.current)
notesCounter.current.innerText = `${count}`;
if (importProgress.current)
importProgress.current.style.width = `${
(filesRead / totalFiles) * 100
}%`;
break;
}
}

4
apps/web/src/stores/attachment-store.js
View File

@@ -24,7 +24,7 @@ import { store as editorStore } from "./editor-store";
import { checkAttachment } from "../common/attachments";
import { showToast } from "../utils/toast";
import { AttachmentStream } from "../utils/streams/attachment-stream";
import { ZipStream } from "../utils/streams/zip-stream";
import { createZipStream } from "../utils/streams/zip-stream";
import { createWriteStream } from "../utils/stream-saver";
let abortController = undefined;
@@ -67,7 +67,7 @@ class AttachmentStore extends BaseStore {
}
);
await attachmentStream
.pipeThrough(new ZipStream())
.pipeThrough(createZipStream(abortController.signal))
.pipeTo(
await createWriteStream("attachments.zip", {
signal: abortController.signal

12
apps/web/src/utils/importer.ts
View File

@@ -27,8 +27,8 @@ import {
ATTACHMENTS_DIRECTORY_NAME,
NOTE_DATA_FILENAME
} from "@notesnook-importer/core/dist/src/utils/note-stream";
import { Reader, Entry } from "./zip-reader";
import { path } from "@notesnook-importer/core/dist/src/utils/path";
import { ZipEntry, createUnzipIterator } from "./streams/unzip-stream";
export async function* importFiles(zipFiles: File[]) {
for (const zip of zipFiles) {
@@ -36,9 +36,8 @@ export async function* importFiles(zipFiles: File[]) {
let filesRead = 0;
const attachments: Record<string, any> = {};
const { read, totalFiles } = await Reader(zip);
for await (const entry of read()) {
for await (const entry of createUnzipIterator(zip)) {
++filesRead;
const isAttachment = entry.name.includes(
@@ -60,7 +59,6 @@ export async function* importFiles(zipFiles: File[]) {
yield {
type: "progress" as const,
count,
totalFiles,
filesRead
};
}
@@ -68,7 +66,7 @@ export async function* importFiles(zipFiles: File[]) {
}
async function processAttachment(
entry: Entry,
entry: ZipEntry,
attachments: Record<string, any>
) {
const name = path.basename(entry.name);
@@ -90,7 +88,7 @@ async function processAttachment(
attachments[name] = { ...cipherData, key };
}
async function processNote(entry: Entry, attachments: Record<string, any>) {
async function processNote(entry: ZipEntry, attachments: Record<string, any>) {
const note = await fileToJson<Note>(entry);
for (const attachment of note.attachments || []) {
const cipherData = attachments[attachment.hash];
@@ -128,7 +126,7 @@ async function processNote(entry: Entry, attachments: Record<string, any>) {
}
}
async function fileToJson<T>(file: Entry) {
async function fileToJson<T>(file: ZipEntry) {
const text = await file.text();
return JSON.parse(text) as T;
}

31
apps/web/src/utils/logger.ts
View File

@@ -23,27 +23,36 @@ import {
logManager
} from "@notesnook/core/dist/logger";
import { LogMessage } from "@notesnook/logger";
import FileSaver from "file-saver";
import { DatabasePersistence, NNStorage } from "../interfaces/storage";
import { zip } from "./zip";
import { ZipFile, createZipStream } from "./streams/zip-stream";
import { createWriteStream } from "./stream-saver";
let logger: typeof _logger;
async function initalizeLogger(persistence: DatabasePersistence = "db") {
initalize(await NNStorage.createInstance("Logs", persistence));
initalize(await NNStorage.createInstance("Logs", persistence), false);
logger = _logger.scope("notesnook-web");
}
async function downloadLogs() {
if (!logManager) return;
const allLogs = await logManager.get();
const files = allLogs.map((log) => ({
filename: log.key,
content: (log.logs as LogMessage[])
.map((line) => JSON.stringify(line))
.join("\n")
}));
const archive = await zip(files, "log");
FileSaver.saveAs(new Blob([archive.buffer]), "notesnook-logs.zip");
const textEncoder = new TextEncoder();
await new ReadableStream<ZipFile>({
pull(controller) {
for (const log of allLogs) {
controller.enqueue({
path: log.key,
data: textEncoder.encode(
(log.logs as LogMessage[])
.map((line) => JSON.stringify(line))
.join("\n")
)
});
}
}
})
.pipeThrough(createZipStream())
.pipeTo(await createWriteStream("notesnook-logs.zip"));
}
async function clearLogs() {

15
apps/web/src/utils/streams/attachment-stream.ts
View File

@@ -19,6 +19,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import { db } from "../../common/db";
import { lazify } from "../lazify";
import { makeUniqueFilename } from "./utils";
import { ZipFile } from "./zip-stream";
export const METADATA_FILENAME = "metadata.json";
@@ -84,17 +85,3 @@ export class AttachmentStream extends ReadableStream<ZipFile> {
});
}
}
function makeUniqueFilename(
filePath: string,
counters: Record<string, number>
) {
filePath = filePath.toLowerCase();
counters[filePath] = (counters[filePath] || 0) + 1;
if (counters[filePath] === 1) return filePath;
const parts = filePath.split(".");
return `${parts.slice(0, -1).join(".")}-${counters[filePath]}.${
parts[parts.length - 1]
}`;
}

115
apps/web/src/utils/streams/export-stream.ts Normal file
View File

@@ -0,0 +1,115 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { sanitizeFilename } from "@notesnook/common";
import { db } from "../../common/db";
import { exportToPDF } from "../../common/export";
import Vault from "../../common/vault";
import { showToast } from "../toast";
import { makeUniqueFilename } from "./utils";
import { ZipFile } from "./zip-stream";
const FORMAT_TO_EXT = {
pdf: "pdf",
md: "md",
txt: "txt",
html: "html",
"md-frontmatter": "md"
} as const;
export class ExportStream extends ReadableStream<ZipFile> {
constructor(
noteIds: string[],
format: "pdf" | "md" | "txt" | "html" | "md-frontmatter",
signal?: AbortSignal,
onProgress?: (current: number, text: string) => void
) {
const textEncoder = new TextEncoder();
let index = 0;
const counters: Record<string, number> = {};
let vaultUnlocked = false;
super({
async start() {
if (noteIds.length === 1 && db.notes?.note(noteIds[0])?.data.locked) {
vaultUnlocked = await Vault.unlockVault();
if (!vaultUnlocked) return false;
} else if (noteIds.length > 1 && (await db.vault?.exists())) {
vaultUnlocked = await Vault.unlockVault();
if (!vaultUnlocked)
showToast(
"error",
"Failed to unlock vault. Locked notes will be skipped."
);
}
},
async pull(controller) {
if (signal?.aborted) {
controller.close();
return;
}
const note = db.notes?.note(noteIds[index++]);
if (!note) return;
if (!vaultUnlocked && note.data.locked) return;
onProgress && onProgress(index, `Exporting "${note.title}"...`);
const rawContent = await db.content?.raw(note.data.contentId);
const content = note.data.locked
? await db.vault?.decryptContent(rawContent)
: rawContent;
const exported = await note
.export(format === "pdf" ? "html" : format, content)
.catch((e: Error) => {
console.error(note.data, e);
showToast(
"error",
`Failed to export note "${note.title}": ${e.message}`
);
});
if (typeof exported !== "string") {
showToast("error", `Failed to export note "${note.title}"`);
return;
}
if (format === "pdf") {
await exportToPDF(note.title, exported);
controller.close();
return;
}
const filename = sanitizeFilename(note.title, { replacement: "-" });
const ext = FORMAT_TO_EXT[format];
controller.enqueue({
path: makeUniqueFilename([filename, ext].join("."), counters),
data: textEncoder.encode(exported),
mtime: new Date(note.data.dateEdited),
ctime: new Date(note.data.dateCreated)
});
if (index === noteIds.length) {
controller.close();
}
}
});
}
}

101
apps/web/src/utils/streams/fflate-shim.js Normal file
View File

@@ -0,0 +1,101 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { Deflate as FflateDeflate, Inflate as FflateInflate } from "fflate";
function initShimAsyncCodec(library, options = {}, registerDataHandler) {
return {
Deflate: createCodecClass(
library.Deflate,
options.deflate,
registerDataHandler
),
Inflate: createCodecClass(
library.Inflate,
options.inflate,
registerDataHandler
)
};
}
function objectHasOwn(object, propertyName) {
// eslint-disable-next-line no-prototype-builtins
return typeof Object.hasOwn === "function"
? Object.hasOwn(object, propertyName)
: // eslint-disable-next-line no-prototype-builtins
object.hasOwnProperty(propertyName);
}
function createCodecClass(
constructor,
constructorOptions,
registerDataHandler
) {
return class {
constructor(options) {
// eslint-disable-next-line @typescript-eslint/no-this-alias
const codecAdapter = this;
const onData = (data) => {
if (codecAdapter.pendingData) {
const previousPendingData = codecAdapter.pendingData;
codecAdapter.pendingData = new Uint8Array(
previousPendingData.length + data.length
);
const { pendingData } = codecAdapter;
pendingData.set(previousPendingData, 0);
pendingData.set(data, previousPendingData.length);
} else {
codecAdapter.pendingData = new Uint8Array(data);
}
};
if (objectHasOwn(options, "level") && options.level === undefined) {
delete options.level;
}
codecAdapter.codec = new constructor(
Object.assign({}, constructorOptions, options)
);
registerDataHandler(codecAdapter.codec, onData);
}
append(data) {
this.codec.push(data);
return getResponse(this);
}
flush() {
this.codec.push(new Uint8Array(), true);
return getResponse(this);
}
};
function getResponse(codec) {
if (codec.pendingData) {
const output = codec.pendingData;
codec.pendingData = null;
return output;
} else {
return new Uint8Array();
}
}
}
const { Deflate, Inflate } = initShimAsyncCodec(
{ Deflate: FflateDeflate, Inflate: FflateInflate },
undefined,
(codec, onData) => (codec.ondata = onData)
);
export { Deflate, Inflate };

77
apps/web/src/utils/streams/unzip-stream.ts Normal file
View File

@@ -0,0 +1,77 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { Deflate, Inflate } from "./fflate-shim";
import {
Entry,
TextWriter,
Uint8ArrayWriter,
ZipReader,
configure
} from "@zip.js/zip.js";
configure({ Deflate, Inflate });
export async function* createUnzipIterator(file: File) {
const reader = new ZipReader(file.stream());
for await (const entry of reader.getEntriesGenerator()) {
yield new ZipEntry(entry);
}
await reader.close();
}
export class ZipEntry extends Blob {
private ts = new TransformStream();
constructor(public readonly entry: Entry) {
super();
}
stream(): ReadableStream<Uint8Array> {
this.entry.getData?.(this.ts.writable);
return this.ts.readable;
}
async text(): Promise<string> {
const writer = new TextWriter("utf-8");
await this.entry.getData?.(writer);
return await writer.getData();
}
get size() {
return this.entry.uncompressedSize;
}
get name() {
return this.entry.filename;
}
async arrayBuffer(): Promise<ArrayBuffer> {
const writer = new Uint8ArrayWriter();
await this.entry.getData?.(writer);
return (await writer.getData()).buffer;
}
slice(
_start?: number | undefined,
_end?: number | undefined,
_contentType?: string | undefined
): Blob {
throw new Error("Slice is not supported.");
}
}

32
apps/web/src/utils/streams/utils.ts Normal file
View File

@@ -0,0 +1,32 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
export function makeUniqueFilename(
filePath: string,
counters: Record<string, number>
) {
filePath = filePath.toLowerCase();
counters[filePath] = (counters[filePath] || 0) + 1;
if (counters[filePath] === 1) return filePath;
const parts = filePath.split(".");
return `${parts.slice(0, -1).join(".")}-${counters[filePath]}.${
parts[parts.length - 1]
}`;
}

71
apps/web/src/utils/streams/zip-stream.ts
View File

@@ -17,32 +17,51 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { Zip, ZipDeflate } from "fflate";
import { Deflate, Inflate } from "./fflate-shim";
import { Uint8ArrayReader, ZipWriter, configure } from "@zip.js/zip.js";
export type ZipFile = { path: string; data: Uint8Array };
export class ZipStream extends TransformStream<ZipFile, Uint8Array> {
constructor() {
const zipper = new Zip();
super({
start(controller) {
zipper.ondata = (err, data) => {
if (err) {
controller.error(err);
} else controller.enqueue(data);
};
},
transform(chunk) {
const fileStream = new ZipDeflate(chunk.path, {
level: 5,
mem: 8
configure({ Deflate, Inflate });
export type ZipFile = {
path: string;
data: Uint8Array;
mtime?: Date;
ctime?: Date;
};
export function createZipStream(signal?: AbortSignal) {
const written = new Set<string>();
const ts = new TransformStream<Uint8Array, Uint8Array>();
const writer = new ZipWriter<Uint8Array>(ts.writable, {
zip64: true,
signal
});
const entryWriter = new WritableStream<ZipFile>({
start() {},
async write(chunk, c) {
// zip.js doesn't support overwriting files.
if (written.has(chunk.path)) return;
await writer
.add(chunk.path, new Uint8ArrayReader(chunk.data), {
creationDate: chunk.ctime,
lastModDate: chunk.mtime
})
.catch(async (e) => {
await ts.writable.abort(e);
await ts.readable.cancel(e);
c.error(e);
});
zipper.add(fileStream);
fileStream.push(chunk.data, true);
},
flush() {
zipper.end();
zipper.terminate();
}
});
}
written.add(chunk.path);
},
async close() {
await writer.close();
await ts.writable.close();
},
async abort(reason) {
await ts.writable.abort(reason);
await ts.readable.cancel(reason);
}
});
return { writable: entryWriter, readable: ts.readable };
}

418
apps/web/src/utils/zip-reader.ts
View File

@@ -1,418 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/* eslint-disable no-underscore-dangle */
/**
* Conflux
* Read (and build) zip files with whatwg streams in the browser.
*
* @author Transcend Inc. <https://transcend.io>
* @license MIT
*/
import { Inflate } from "fflate";
const ERR_BAD_FORMAT = "File format is not recognized.";
const ZIP_COMMENT_MAX = 65536;
const EOCDR_MIN = 22;
const EOCDR_MAX = EOCDR_MIN + ZIP_COMMENT_MAX;
const MAX_VALUE_32BITS = 0xffffffff;
const decoder = new TextDecoder();
const uint16e = (b: Uint8Array, n: number) => b[n] | (b[n + 1] << 8);
export class Entry {
private dataView: DataView;
private _fileLike: File;
private _extraFields: Record<string, DataView>;
constructor(dataView: DataView, fileLike: File) {
if (dataView.getUint32(0) !== 0x504b0102) {
throw new Error("ERR_BAD_FORMAT");
}
const dv = dataView;
this.dataView = dv;
this._fileLike = fileLike;
this._extraFields = {};
for (let i = 46 + this.filenameLength; i < dv.byteLength; ) {
const id = dv.getUint16(i, true);
const len = dv.getUint16(i + 2, true);
const start = dv.byteOffset + i + 4;
this._extraFields[id] = new DataView(dv.buffer.slice(start, start + len));
i += len + 4;
}
}
get webkitRelativePath(): string {
return this.name;
}
get type(): string {
return "application/octet-stream";
}
get versionMadeBy() {
return this.dataView.getUint16(4, true);
}
get versionNeeded() {
return this.dataView.getUint16(6, true);
}
get bitFlag() {
return this.dataView.getUint16(8, true);
}
get encrypted() {
return (this.bitFlag & 0x0001) === 0x0001;
}
get compressionMethod() {
return this.dataView.getUint16(10, true);
}
get crc32() {
return this.dataView.getUint32(16, true);
}
get compressedSize() {
return this.dataView.getUint32(20, true);
}
get filenameLength() {
return this.dataView.getUint16(28, true);
}
get extraFieldLength() {
return this.dataView.getUint16(30, true);
}
get commentLength() {
return this.dataView.getUint16(32, true);
}
get diskNumberStart() {
return this.dataView.getUint16(34, true);
}
get internalFileAttributes() {
return this.dataView.getUint16(36, true);
}
get externalFileAttributes() {
return this.dataView.getUint32(38, true);
}
get directory() {
return !!(this.dataView.getUint8(38) & 16);
}
get offset() {
return this.dataView.getUint32(42, true);
}
get zip64() {
return this.dataView.getUint32(24, true) === MAX_VALUE_32BITS;
}
get comment() {
const dv = this.dataView;
const uint8 = new Uint8Array(
dv.buffer,
dv.byteOffset + this.filenameLength + this.extraFieldLength + 46,
this.commentLength
);
return decoder.decode(uint8);
}
// File like IDL methods
get lastModifiedDate() {
const t = this.dataView.getUint32(12, true);
return new Date(
// Date.UTC(
((t >> 25) & 0x7f) + 1980, // year
((t >> 21) & 0x0f) - 1, // month
(t >> 16) & 0x1f, // day
(t >> 11) & 0x1f, // hour
(t >> 5) & 0x3f, // minute
(t & 0x1f) << 1
// ),
);
}
get lastModified() {
return +this.lastModifiedDate;
}
get name() {
if (!this.bitFlag && this._extraFields && this._extraFields[0x7075]) {
return decoder.decode(this._extraFields[0x7075].buffer.slice(5));
}
const dv = this.dataView;
const uint8 = new Uint8Array(
dv.buffer,
dv.byteOffset + 46,
this.filenameLength
);
return decoder.decode(uint8);
}
get size() {
const size = this.dataView.getUint32(24, true);
return size === MAX_VALUE_32BITS ? this._extraFields[1].getUint8(0) : size;
}
stream() {
// eslint-disable-next-line @typescript-eslint/no-this-alias
const self = this;
const crc = new Crc32();
let inflator: Inflate | undefined;
const onEnd = (ctrl: ReadableStreamController<Uint8Array>) =>
crc.get() === self.crc32
? ctrl.close()
: ctrl.error(new Error("The crc32 checksum don't match"));
return new ReadableStream<Uint8Array>({
async start(ctrl) {
// Need to read local header to get fileName + extraField length
// Since they are not always the same length as in central dir...
const ab = await self._fileLike
.slice(self.offset + 26, self.offset + 30)
.arrayBuffer();
const bytes = new Uint8Array(ab);
const localFileOffset = uint16e(bytes, 0) + uint16e(bytes, 2) + 30;
const start = self.offset + localFileOffset;
const end = start + self.compressedSize;
(this as any).reader = self._fileLike
.slice(start, end)
.stream()
.getReader();
if (self.compressionMethod) {
inflator = new Inflate();
inflator.ondata = (chunk, final) => {
crc.append(chunk);
ctrl.enqueue(chunk);
if (final) onEnd(ctrl);
};
}
},
async pull(ctrl) {
const v = await (this as any).reader.read();
if (inflator && !v.done) {
inflator.push(v.value);
} else if (v.done) {
onEnd(ctrl);
} else {
ctrl.enqueue(v.value);
crc.append(v.value);
}
}
});
}
arrayBuffer() {
return new Response(this.stream()).arrayBuffer().catch((e) => {
throw new Error(`Failed to read Entry\n${e}`);
});
}
text() {
return new Response(this.stream()).text().catch((e) => {
throw new Error(`Failed to read Entry\n${e}`);
});
}
file() {
return new Response(this.stream())
.blob()
.then(
(blob) =>
new File([blob], this.name, { lastModified: this.lastModified })
)
.catch((e) => {
throw new Error(`Failed to read Entry\n${e}`);
});
}
}
/**
* Get a BigInt 64 from a DataView
*
* @param {DataView} view a dataview
* @param {number} position the position
* @param {boolean} littleEndian whether this uses littleEndian encoding
* @returns BigInt
*/
function getBigInt64(view: DataView, position: number, littleEndian = false) {
if ("getBigInt64" in DataView.prototype) {
return view.getBigInt64(position, littleEndian);
}
let value = BigInt(0);
const isNegative =
(view.getUint8(position + (littleEndian ? 7 : 0)) & 0x80) > 0;
let carrying = true;
for (let i = 0; i < 8; i++) {
let byte = view.getUint8(position + (littleEndian ? i : 7 - i));
if (isNegative) {
if (carrying) {
if (byte !== 0x00) {
byte = ~(byte - 1) & 0xff;
carrying = false;
}
} else {
byte = ~byte & 0xff;
}
}
value = value + BigInt(byte) + BigInt(256) ** BigInt(i);
}
if (isNegative) {
value = -value;
}
return value;
}
export async function Reader(file: File) {
// Seek EOCDR - "End of central directory record" is the last part of a zip archive, and is at least 22 bytes long.
// Zip file comment is the last part of EOCDR and has max length of 64KB,
// so we only have to search the last 64K + 22 bytes of a archive for EOCDR signature (0x06054b50).
if (file.size < EOCDR_MIN) throw new Error(ERR_BAD_FORMAT);
// seek last length bytes of file for EOCDR
async function doSeek(length: number) {
const ab = await file.slice(file.size - length).arrayBuffer();
const bytes = new Uint8Array(ab);
for (let i = bytes.length - EOCDR_MIN; i >= 0; i--) {
if (
bytes[i] === 0x50 &&
bytes[i + 1] === 0x4b &&
bytes[i + 2] === 0x05 &&
bytes[i + 3] === 0x06
) {
return new DataView(bytes.buffer, i, EOCDR_MIN);
}
}
return null;
}
// In most cases, the EOCDR is EOCDR_MIN bytes long
let dv =
(await doSeek(EOCDR_MIN)) || (await doSeek(Math.min(EOCDR_MAX, file.size)));
if (!dv) throw new Error(ERR_BAD_FORMAT);
let fileslength = dv.getUint16(8, true);
let centralDirSize = dv.getUint32(12, true);
let centralDirOffset = dv.getUint32(16, true);
// const fileCommentLength = dv.getUint16(20, true);
const isZip64 = centralDirOffset === MAX_VALUE_32BITS;
if (isZip64) {
const l = -dv.byteLength - 20;
dv = new DataView(await file.slice(l, -dv.byteLength).arrayBuffer());
// const signature = dv.getUint32(0, true) // 4 bytes
// const diskWithZip64CentralDirStart = dv.getUint32(4, true) // 4 bytes
const relativeOffsetEndOfZip64CentralDir = Number(getBigInt64(dv, 8, true)); // 8 bytes
// const numberOfDisks = dv.getUint32(16, true) // 4 bytes
const zip64centralBlob = file.slice(relativeOffsetEndOfZip64CentralDir, l);
dv = new DataView(await zip64centralBlob.arrayBuffer());
// const zip64EndOfCentralSize = dv.getBigInt64(4, true)
// const diskNumber = dv.getUint32(16, true)
// const diskWithCentralDirStart = dv.getUint32(20, true)
// const centralDirRecordsOnThisDisk = dv.getBigInt64(24, true)
fileslength = Number(getBigInt64(dv, 32, true));
centralDirSize = Number(getBigInt64(dv, 40, true));
centralDirOffset = Number(getBigInt64(dv, 48, true));
}
if (centralDirOffset < 0 || centralDirOffset >= file.size) {
throw new Error(ERR_BAD_FORMAT);
}
const start = centralDirOffset;
const end = centralDirOffset + centralDirSize;
const blob = file.slice(start, end);
const bytes = new Uint8Array(await blob.arrayBuffer());
async function* read() {
for (let i = 0, index = 0; i < fileslength; i++) {
const size =
uint16e(bytes, index + 28) + // filenameLength
uint16e(bytes, index + 30) + // extraFieldLength
uint16e(bytes, index + 32) + // commentLength
46;
if (index + size > bytes.length) {
throw new Error("Invalid ZIP file.");
}
yield new Entry(new DataView(bytes.buffer, index, size), file);
index += size;
}
}
return { read, totalFiles: fileslength };
}
/* eslint-disable no-param-reassign */
class Crc32 {
crc: number;
constructor() {
this.crc = -1;
}
append(data: Uint8Array) {
let crc = this.crc | 0;
const { table } = this;
for (let offset = 0, len = data.length | 0; offset < len; offset++) {
crc = (crc >>> 8) ^ table[(crc ^ data[offset]) & 0xff];
}
this.crc = crc;
}
get() {
return (this.crc ^ -1) >>> 0;
}
private table = ((table: number[], i: number, j: number, t: number) => {
for (i = 0; i < 256; i++) {
t = i;
for (j = 0; j < 8; j++) {
t = t & 1 ? (t >>> 1) ^ 0xedb88320 : t >>> 1;
}
table[i] = t;
}
return table;
})([], 0, 0, 0);
}

48
apps/web/src/utils/zip.ts
View File

@@ -1,48 +0,0 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { AsyncZippable, zip as zipAsync } from "fflate";
import { sanitizeFilename } from "@notesnook/common";
const textEncoder = new TextEncoder();
type File = { filename: string; content: string; date: number };
async function zip(files: File[], format: string): Promise<Uint8Array> {
const obj: AsyncZippable = Object.create(null);
files.forEach((file) => {
const name = sanitizeFilename(file.filename, { replacement: "-" });
let counter = 0;
while (obj[makeFilename(name, format, counter)]) ++counter;
obj[makeFilename(name, format, counter)] = [
textEncoder.encode(file.content),
{ mtime: file.date }
];
});
return new Promise((resolve, reject) =>
zipAsync(obj, (err, data) => (err ? reject(err) : resolve(data)))
);
}
export { zip };
function makeFilename(filename: string, extension: string, counter: number) {
let final = filename;
if (counter) final += `-${counter}`;
final += `.${extension}`;
return final;
}

2
packages/core/src/api/vault.js
View File

@@ -210,7 +210,7 @@ export default class Vault {
return await this._lockNote(note, this._password);
}
async exists(vaultKey) {
async exists(vaultKey = undefined) {
if (!vaultKey) vaultKey = await this._getKey();
return vaultKey && vaultKey.cipher && vaultKey.iv;
}