Compare commits

...

3 Commits

Author SHA1 Message Date
Abdullah Atta
945fd9c091 web: add ui to view failed sync items 2026-08-11 12:22:01 +05:00
Abdullah Atta
0c95359e1d core: add handling for failed sync items 2026-08-10 22:54:07 +05:00
Abdullah Atta
1d86e359a9 core: try to decrypt items with unknown key version with all available keys 2026-08-03 14:21:00 +05:00
13 changed files with 1161 additions and 41 deletions

View File

@@ -0,0 +1,345 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { useState } from "react";
import { getFormattedDate, usePromise } from "@notesnook/common";
import { FailedSyncItem } from "@notesnook/core";
import { SerializedKey } from "@notesnook/crypto";
import { Box, Button, Flex, Text } from "@theme-ui/components";
import { db } from "../common/db";
import { BaseDialogProps, DialogManager } from "../common/dialog-manager";
import Dialog from "../components/dialog";
import { FlexScrollContainer } from "../components/scroll-container";
import { strings } from "@notesnook/intl";
import { showToast } from "../utils/toast";
import { ConfirmDialog } from "./confirm";
import { showPasswordDialog } from "./password-dialog";
type FailedSyncItemsDialogProps = BaseDialogProps<boolean>;
const COLUMNS = [
{ title: strings.dateSynced(), width: "150px" },
{ title: strings.dataTypesCamelCase.item(), width: "110px" },
{ title: strings.itemId(), width: "140px" },
// { title: strings.keyVersion(), width: "90px" },
{ title: strings.error(), width: "1fr" }
];
function TypeBadge({ type }: { type: string }) {
return (
<Text
sx={{
bg: "background-secondary",
color: "paragraph",
borderRadius: "default",
px: "6px",
py: "2px",
fontSize: "0.65em",
fontWeight: 600,
textTransform: "uppercase",
whiteSpace: "nowrap"
}}
>
{type}
</Text>
);
}
async function resolveKey(input: {
password?: string;
key?: string;
}): Promise<SerializedKey> {
const user = await db.user.getUser();
if (!user?.salt) throw new Error("User salt not found. Please relogin.");
if (input.key) {
return { key: input.key, salt: user.salt };
}
if (!input.password) throw new Error("Password or encryption key required.");
try {
return await db.storage().generateCryptoKey(input.password, user.salt);
} catch {
return await db
.storage()
.generateCryptoKeyFallback(input.password, user.salt);
}
}
async function retryWithKey(
ids: string[],
key: SerializedKey
): Promise<boolean> {
const result = await db.syncer.sync.retryFailedItems(ids, key);
if (result.succeeded.length > 0 && result.failed.length === 0) {
showToast("success", strings.decryptionSucceeded(result.succeeded.length));
return true;
}
if (result.succeeded.length > 0 && result.failed.length > 0) {
showToast(
"error",
strings.decryptionPartialSuccess(
result.succeeded.length,
result.failed.length
)
);
// Still close the password dialog so the table can refresh.
return true;
}
const firstError = result.failed[0]?.error;
throw new Error(
firstError
? `${strings.decryptionFailed()}: ${firstError}`
: strings.decryptionFailed()
);
}
export const FailedSyncItemsDialog = DialogManager.register(
function FailedSyncItemsDialog(props: FailedSyncItemsDialogProps) {
const result = usePromise(() => db.failedSyncItems.all.items());
const [busyIds, setBusyIds] = useState<string[]>([]);
function refresh() {
if (result.status !== "pending") result.refresh();
}
async function deleteAll() {
const ok = await ConfirmDialog.show({
title: strings.deleteAll(),
subtitle: strings.deleteAllFailedSyncItemsDesc(),
positiveButtonText: strings.yes(),
negativeButtonText: strings.no()
});
if (!ok) return;
await db.failedSyncItems.clear();
showToast("success", strings.allItemsDeleted());
refresh();
}
async function retryWithCurrentKeys(ids: string[]) {
setBusyIds((prev) => [...prev, ...ids]);
try {
const keys = await db.user.getDataEncryptionKeys();
if (!keys?.length) {
showToast("error", strings.decryptionFailed());
return;
}
let succeeded = 0;
let remaining = [...ids];
for (const keyInfo of keys) {
if (remaining.length === 0) break;
const result = await db.syncer.sync.retryFailedItems(
remaining,
keyInfo.key
);
succeeded += result.succeeded.length;
remaining = result.failed.map((f) => f.id);
}
if (succeeded > 0 && remaining.length === 0) {
showToast("success", strings.decryptionSucceeded(succeeded));
} else if (succeeded > 0) {
showToast(
"error",
strings.decryptionPartialSuccess(succeeded, remaining.length)
);
} else {
showToast("error", strings.decryptionFailed());
}
refresh();
} finally {
setBusyIds((prev) => prev.filter((id) => !ids.includes(id)));
}
}
async function retryWithCustomKey(ids: string[]) {
const ok = await showPasswordDialog({
title: strings.retryWithCustomKey(),
inputs: {
key: {
label: strings.encryptionKey(),
type: "password",
required: true
}
},
async validate({ key }) {
if (!key) return false;
setBusyIds((prev) => [...prev, ...ids]);
try {
const resolved = await resolveKey({ key });
return await retryWithKey(ids, resolved);
} finally {
setBusyIds((prev) => prev.filter((id) => !ids.includes(id)));
}
}
});
if (ok) refresh();
}
const items: FailedSyncItem[] =
result.status === "fulfilled" ? result.value : [];
const allIds = items.map((item) => item.id);
const hasItems = items.length > 0;
return (
<Dialog
isOpen={true}
title={strings.failedSyncItems()}
description={strings.failedSyncItemsDesc()}
titleAction={
hasItems ? (
<Flex sx={{ gap: 1 }}>
<Button
variant="secondary"
onClick={() => retryWithCurrentKeys(allIds)}
disabled={busyIds.length > 0}
>
{strings.retryWithCurrentKeys()}
</Button>
<Button
variant="secondary"
onClick={() => retryWithCustomKey(allIds)}
disabled={busyIds.length > 0}
>
{strings.retryWithCustomKey()}
</Button>
<Button variant="errorSecondary" onClick={deleteAll}>
{strings.deleteAll()}
</Button>
</Flex>
) : undefined
}
onClose={() => props.onClose(false)}
negativeButton={{
text: strings.close(),
onClick: () => props.onClose(false)
}}
noScroll
width="900px"
>
{result.status === "pending" ? (
<Text sx={{ p: 3 }} variant="body">
{strings.loading()}
</Text>
) : result.status === "rejected" ? (
<Text sx={{ p: 3 }} variant="body">
{strings.failed()}
</Text>
) : !hasItems ? (
<Text sx={{ p: 3 }} variant="body">
{strings.noFailedSyncItems()}
</Text>
) : (
<FlexScrollContainer style={{ maxHeight: "70vh" }}>
<Box sx={{ p: 2 }}>
<Box
as="table"
sx={{
width: "100%",
borderCollapse: "collapse",
tableLayout: "fixed",
"th, td": {
px: 2,
py: 1,
textAlign: "left",
verticalAlign: "top",
borderBottom: "1px solid var(--separator)"
}
}}
>
<Box as="thead">
<Box as="tr">
{COLUMNS.map((col) => (
<Box
key={col.title || "actions"}
as="th"
sx={{ width: col.width, whiteSpace: "nowrap" }}
>
<Text variant="subtitle">{col.title}</Text>
</Box>
))}
</Box>
</Box>
<Box as="tbody">
{items.map((item) => {
const isBusy = busyIds.includes(item.id);
const errors = item.errors ?? [];
return (
<Box as="tr" key={item.id}>
<Box as="td">
<Text variant="body" sx={{ whiteSpace: "nowrap" }}>
{getFormattedDate(item.dateSynced)}
</Text>
</Box>
<Box as="td">
<TypeBadge type={item.itemType} />
</Box>
<Box as="td">
<Text
variant="body"
className="selectable"
sx={{
fontFamily: "monospace",
fontSize: "0.8em",
wordBreak: "break-all"
}}
title={item.itemId}
>
{item.itemId}
</Text>
</Box>
<Box as="td">
{errors.length > 0 ? (
<Box
as="pre"
className="selectable"
sx={{
color: "paragraph",
m: 0,
fontSize: "0.72em",
whiteSpace: "pre-wrap",
wordBreak: "break-all"
}}
>
{errors.join("\n")}
</Box>
) : (
<Text
variant="body"
sx={{ color: "paragraph-secondary" }}
>
</Text>
)}
</Box>
</Box>
);
})}
</Box>
</Box>
</Box>
</FlexScrollContainer>
)}
</Dialog>
);
}
);

View File

@@ -22,6 +22,7 @@ import { useStore as useAppStore } from "../../stores/app-store";
import { useStore as useSettingStore } from "../../stores/setting-store";
import { ConfirmDialog } from "../confirm";
import { strings } from "@notesnook/intl";
import { FailedSyncItemsDialog } from "../failed-sync-items-dialog";
export const SyncSettings: SettingsGroup[] = [
{
@@ -92,7 +93,14 @@ export const SyncSettings: SettingsGroup[] = [
toggle: () => useSettingStore.getState().toggleFullOfflineMode()
}
]
},
}
]
},
{
key: "troubleshoot-sync",
section: "sync",
header: strings.troubleshoot(),
settings: [
{
key: "force-sync",
title: strings.havingProblemsWithSync(),
@@ -140,6 +148,28 @@ export const SyncSettings: SettingsGroup[] = [
})
}
]
},
{
key: "failed-sync-items",
title: strings.failedSyncItems(),
description: strings.failedSyncItemsDesc(),
keywords: [
"failed",
"decrypt",
"sync error",
"missing key",
"ciphertext"
],
components: [
{
type: "button",
title: strings.show(),
variant: "secondary",
action: () => {
FailedSyncItemsDialog.show({});
}
}
]
}
]
}

View File

@@ -85,6 +85,7 @@ import { LazyPromise } from "../utils/lazy-promise.js";
import { InboxApiKeys } from "./inbox-api-keys.js";
import { Circle } from "./circle.js";
import { Wrapped } from "./wrapped.js";
import { FailedSyncItems } from "../collections/failed-sync-items.js";
type EventSourceConstructor = new (
uri: string,
@@ -230,6 +231,7 @@ class Database {
inboxApiKeys = new InboxApiKeys(this, this.tokenManager);
inboxItemsHistory = new InboxItemsHistory(this);
failedSyncItems = new FailedSyncItems(this);
wrapped = new Wrapped(this);
@@ -353,6 +355,7 @@ class Database {
await this.monographsCollection.init();
await this.inboxItemsHistory.init();
await this.failedSyncItems.init();
await this.trash.init();

View File

@@ -0,0 +1,383 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import {
databaseTest,
loginFakeUser,
TEST_NOTE
} from "../../../../__tests__/utils/index.ts";
import { KEY_VERSION } from "../types.ts";
import { test, expect, vi } from "vitest";
const CURRENT_DATABASE_VERSION = 6.1;
function chunkItem(id, keyVersion) {
return {
id,
v: CURRENT_DATABASE_VERSION,
keyVersion,
format: "base64",
alg: "xchacha20-poly1305",
cipher: "ciphertext",
iv: "iv",
salt: "salt",
length: 1
};
}
function decryptFailure() {
return Promise.reject(
new Error("ciphertext cannot be decrypted using that key")
);
}
function failedItemPayload(itemId, itemType) {
return {
itemId,
itemType,
cipher: chunkItem(itemId, KEY_VERSION.LEGACY),
errors: [`Failed to decrypt item ${itemId} with key version 0.`],
dateSynced: Date.now()
};
}
test("items with a key version that has no matching key are stored in failedSyncItems", () =>
databaseTest().then(async (db) => {
await loginFakeUser(db);
const item = chunkItem("setting-1", KEY_VERSION.DEK);
const keys = [{ version: KEY_VERSION.LEGACY, key: { key: "legacy-key" } }];
// decryptMulti is never called for the DEK version group (no matching key),
// so only the per-item retry path runs. Make all retries fail.
vi.spyOn(db.storage(), "decrypt").mockImplementation(decryptFailure);
await db.syncer.sync.processChunk(
{ type: "settingitem", count: 1, items: [item] },
keys,
{ type: "fetch" }
);
const failedItems = await db.failedSyncItems.all.items();
expect(failedItems).toHaveLength(1);
expect(failedItems[0].itemId).toBe(item.id);
expect(failedItems[0].itemType).toBe("settingitem");
expect(failedItems[0].cipher.id).toBe(item.id);
expect(failedItems[0].errors.length).toBe(keys.length);
expect(failedItems[0].errors[0]).toContain(
"ciphertext cannot be decrypted"
);
}));
test("items that fail to decrypt with all available keys are stored in failedSyncItems", () =>
databaseTest().then(async (db) => {
await loginFakeUser(db);
const item = chunkItem("setting-1", KEY_VERSION.LEGACY);
const keys = [
{ version: KEY_VERSION.LEGACY, key: { key: "legacy-key" } },
{ version: KEY_VERSION.DEK, key: { key: "dek-key" } }
];
vi.spyOn(db.storage(), "decryptMulti").mockImplementation(decryptFailure);
vi.spyOn(db.storage(), "decrypt").mockImplementation(decryptFailure);
await db.syncer.sync.processChunk(
{ type: "settingitem", count: 1, items: [item] },
keys,
{ type: "fetch" }
);
const failedItems = await db.failedSyncItems.all.items();
expect(failedItems).toHaveLength(1);
expect(failedItems[0].errors.length).toBe(keys.length);
expect(failedItems[0].errors[0]).toContain("key version 0");
expect(failedItems[0].errors[1]).toContain("key version 1");
}));
test("items that fail batch decryption but succeed on a fallback key are decrypted and not stored as failed", () =>
databaseTest().then(async (db) => {
await loginFakeUser(db);
const item = chunkItem("setting-1", KEY_VERSION.DEK);
const keys = [
{ version: KEY_VERSION.LEGACY, key: { key: "legacy-key" } },
{ version: KEY_VERSION.DEK, key: { key: "dek-key" } }
];
// Batch decryption with the DEK key fails...
vi.spyOn(db.storage(), "decryptMulti").mockImplementation(decryptFailure);
// ...but the per-item retry succeeds with the legacy key.
const decryptedItem = JSON.stringify({
id: item.id,
type: "settingitem",
deleted: true,
dateCreated: Date.now(),
dateModified: Date.now()
});
vi.spyOn(db.storage(), "decrypt").mockImplementation((key) =>
key === keys[0].key ? Promise.resolve(decryptedItem) : decryptFailure()
);
await db.syncer.sync.processChunk(
{ type: "settingitem", count: 1, items: [item] },
keys,
{ type: "fetch" }
);
const failedItems = await db.failedSyncItems.all.items();
expect(failedItems).toHaveLength(0);
expect(db.storage().decrypt).toHaveBeenCalledTimes(1);
}));
test("successfully decrypted items are not stored in failedSyncItems", () =>
databaseTest().then(async (db) => {
await loginFakeUser(db);
const item = chunkItem("setting-1", KEY_VERSION.LEGACY);
const keys = [{ version: KEY_VERSION.LEGACY, key: { key: "legacy-key" } }];
const decryptedItem = JSON.stringify({
id: item.id,
type: "settingitem",
deleted: true,
dateCreated: Date.now(),
dateModified: Date.now()
});
vi.spyOn(db.storage(), "decryptMulti").mockResolvedValue([decryptedItem]);
await db.syncer.sync.processChunk(
{ type: "settingitem", count: 1, items: [item] },
keys,
{ type: "fetch" }
);
const failedItems = await db.failedSyncItems.all.items();
expect(failedItems).toHaveLength(0);
}));
test("failedSyncItems.add stores an item and assigns it a new id", () =>
databaseTest().then(async (db) => {
await loginFakeUser(db);
const itemId = await db.notes.add(TEST_NOTE);
const id = await db.failedSyncItems.add(failedItemPayload(itemId, "note"));
expect(id).toBeDefined();
expect(id).not.toBe(itemId);
const items = await db.failedSyncItems.all.items();
expect(items).toHaveLength(1);
expect(items[0].itemId).toBe(itemId);
expect(items[0].itemType).toBe("note");
expect(items[0].cipher.id).toBe(itemId);
expect(items[0].errors).toEqual([
`Failed to decrypt item ${itemId} with key version 0.`
]);
}));
test("failedSyncItems.delete removes the failed item and soft-deletes the underlying item", () =>
databaseTest().then(async (db) => {
await loginFakeUser(db);
const noteId = await db.notes.add(TEST_NOTE);
const failedId = await db.failedSyncItems.add(
failedItemPayload(noteId, "note")
);
await db.failedSyncItems.delete([failedId]);
const failedItems = await db.failedSyncItems.all.items();
expect(failedItems).toHaveLength(0);
expect(await isItemDeleted(db, "notes", noteId)).toBe(true);
}));
test("failedSyncItems.delete groups items by type and soft-deletes in the right collections", () =>
databaseTest().then(async (db) => {
await loginFakeUser(db);
const noteId = await db.notes.add(TEST_NOTE);
const tagId = await db.tags.add({ title: "test-tag" });
const failedNoteId = await db.failedSyncItems.add(
failedItemPayload(noteId, "note")
);
const failedTagId = await db.failedSyncItems.add(
failedItemPayload(tagId, "tag")
);
await db.failedSyncItems.delete([failedNoteId, failedTagId]);
expect(await db.failedSyncItems.all.items()).toHaveLength(0);
expect(await isItemDeleted(db, "notes", noteId)).toBe(true);
expect(await isItemDeleted(db, "tags", tagId)).toBe(true);
}));
test("failedSyncItems.delete only deletes the requested ids", () =>
databaseTest().then(async (db) => {
await loginFakeUser(db);
const noteId1 = await db.notes.add(TEST_NOTE);
const noteId2 = await db.notes.add(TEST_NOTE);
const failedId1 = await db.failedSyncItems.add(
failedItemPayload(noteId1, "note")
);
await db.failedSyncItems.add(failedItemPayload(noteId2, "note"));
await db.failedSyncItems.delete([failedId1]);
const failedItems = await db.failedSyncItems.all.items();
expect(failedItems).toHaveLength(1);
expect(failedItems[0].itemId).toBe(noteId2);
expect(await isItemDeleted(db, "notes", noteId1)).toBe(true);
expect(await db.notes.collection.get(noteId2)).toBeDefined();
}));
test("failedSyncItems.delete skips unknown collection types gracefully", () =>
databaseTest().then(async (db) => {
await loginFakeUser(db);
const failedId = await db.failedSyncItems.add(
failedItemPayload("unknown-id", "unknown-type")
);
// Unknown type has no entry in SYNC_COLLECTIONS_MAP so it's skipped
// gracefully; the failed item row itself is still removed.
await expect(db.failedSyncItems.delete([failedId])).resolves.not.toThrow();
expect(await db.failedSyncItems.all.items()).toHaveLength(0);
}));
test("failedSyncItems.clear removes all failed items", () =>
databaseTest().then(async (db) => {
await loginFakeUser(db);
const noteId1 = await db.notes.add(TEST_NOTE);
const noteId2 = await db.notes.add(TEST_NOTE);
await db.failedSyncItems.add(failedItemPayload(noteId1, "note"));
await db.failedSyncItems.add(failedItemPayload(noteId2, "note"));
await db.failedSyncItems.clear();
expect(await db.failedSyncItems.all.items()).toHaveLength(0);
expect(await isItemDeleted(db, "notes", noteId1)).toBeTruthy();
expect(await isItemDeleted(db, "notes", noteId2)).toBeTruthy();
}));
test("failedSyncItems.remove deletes the failed row without soft-deleting the underlying item", () =>
databaseTest().then(async (db) => {
await loginFakeUser(db);
const noteId = await db.notes.add(TEST_NOTE);
const failedId = await db.failedSyncItems.add(
failedItemPayload(noteId, "note")
);
await db.failedSyncItems.remove([failedId]);
expect(await db.failedSyncItems.all.items()).toHaveLength(0);
expect(await isItemDeleted(db, "notes", noteId)).toBe(false);
expect(await db.notes.note(noteId)).toBeDefined();
}));
test("retryFailedItems decrypts with a custom key, merges the item, and removes the failed row", () =>
databaseTest().then(async (db) => {
await loginFakeUser(db);
const keys = await db.user.getDataEncryptionKeys();
const key = keys[0].key;
const notePayload = {
id: "retry-note-1",
type: "note",
title: "Recovered Note",
dateModified: Date.now(),
dateCreated: Date.now()
};
const cipher = await db.storage().encrypt(key, JSON.stringify(notePayload));
const failedId = await db.failedSyncItems.add({
itemId: notePayload.id,
itemType: "note",
cipher: {
...cipher,
id: notePayload.id,
v: CURRENT_DATABASE_VERSION,
keyVersion: keys[0].version
},
errors: ["previous failure"],
dateSynced: Date.now()
});
const result = await db.syncer.sync.retryFailedItems([failedId], key);
expect(result.succeeded).toEqual([failedId]);
expect(result.failed).toHaveLength(0);
expect(await db.failedSyncItems.all.items()).toHaveLength(0);
const note = await db.notes.note(notePayload.id);
expect(note).toBeDefined();
expect(note.title).toBe("Recovered Note");
}));
test("retryFailedItems records a failure and keeps the failed row when the key is wrong", () =>
databaseTest().then(async (db) => {
await loginFakeUser(db);
const keys = await db.user.getDataEncryptionKeys();
const key = keys[0].key;
const wrongKey = await db.crypto().generateRandomKey();
const notePayload = {
id: "retry-note-2",
type: "note",
title: "Still Locked",
dateModified: Date.now(),
dateCreated: Date.now()
};
const cipher = await db.storage().encrypt(key, JSON.stringify(notePayload));
const failedId = await db.failedSyncItems.add({
itemId: notePayload.id,
itemType: "note",
cipher: {
...cipher,
id: notePayload.id,
v: CURRENT_DATABASE_VERSION,
keyVersion: keys[0].version
},
errors: ["previous failure"],
dateSynced: Date.now()
});
const result = await db.syncer.sync.retryFailedItems([failedId], wrongKey);
expect(result.succeeded).toHaveLength(0);
expect(result.failed).toHaveLength(1);
expect(result.failed[0].id).toBe(failedId);
const failedItems = await db.failedSyncItems.all.items();
expect(failedItems).toHaveLength(1);
expect(failedItems[0].errors.length).toBeGreaterThan(1);
expect(await db.notes.note(notePayload.id)).toBeUndefined();
}));
async function isItemDeleted(db, collectionName, itemId) {
const item = await db
.sql()
.selectFrom(collectionName)
.selectAll()
.where("id", "==", itemId)
.executeTakeFirst();
return item?.deleted === true;
}

View File

@@ -51,6 +51,7 @@ import {
SYNC_COLLECTIONS_MAP,
SyncableItemType,
SyncInboxItem,
SyncItem,
SyncTransferItem
} from "./types.js";
import { DownloadableFile } from "../../database/fs.js";
@@ -366,11 +367,10 @@ export class Sync {
options: SyncOptions
) {
const itemType = chunk.type;
const decrypted: string[] = [];
const decrypted: { data: string; version: number }[] = [];
// Pre-group items by keyVersion for O(1) lookups
const itemsByKeyVersion = new Map<KeyVersion, typeof chunk.items>();
const versionMap = new Map<string, number>();
for (const item of chunk.items) {
const keyVersion = item.keyVersion ?? KEY_VERSION.LEGACY;
@@ -380,34 +380,156 @@ export class Sync {
} else {
itemsByKeyVersion.set(keyVersion, [item]);
}
versionMap.set(item.id, item.v);
}
for (const keyInfo of keys) {
const itemsToDecrypt = itemsByKeyVersion.get(keyInfo.version);
if (!itemsToDecrypt || itemsToDecrypt.length === 0) continue;
const failedItems: SyncItem[] = [];
for (const [keyVersion, items] of itemsByKeyVersion.entries()) {
const keyInfo = keys.find((k) => k.version === keyVersion);
if (!keyInfo) {
this.logger.error(
new Error(
`No key found for key version ${keyVersion}. Retrying items will all available keys.`
)
);
failedItems.push(...items);
continue;
}
this.logger.info("Decrypting using key", {
keyInfo: keyInfo.version,
items: itemsToDecrypt.length
items: items.length
});
decrypted.push(
...(await this.db.storage().decryptMulti(keyInfo.key, itemsToDecrypt))
);
try {
const decryptedItems = await this.db
.storage()
.decryptMulti(keyInfo.key, items);
for (let i = 0; i < decryptedItems.length; ++i) {
decrypted.push({ data: decryptedItems[i], version: items[i].v });
}
} catch (error) {
this.logger.error(
error,
`Failed to decrypt items with key version ${keyInfo.version}.`
);
failedItems.push(...items);
}
}
const deserialized: MaybeDeletedItem<Item>[] = [];
for (let i = 0; i < decrypted.length; ++i) {
const decryptedItem = JSON.parse(decrypted[i]) as MaybeDeletedItem<Item>;
const version = versionMap.get(decryptedItem.id);
if (version === undefined) {
this.logger.error(
new Error(
`Version not found for item ${decryptedItem.id}. Skipping item.`
)
);
continue;
if (failedItems.length > 0) {
for (const item of failedItems) {
const decryptionErrors: string[] = [];
for (const keyInfo of keys) {
this.logger.info("Decrypting failed sync item using key", {
keyInfo: keyInfo.version
});
const decryptedItem = await this.db
.storage()
.decrypt(keyInfo.key, item)
.catch((error) => {
decryptionErrors.push(
error instanceof Error ? error.message : JSON.stringify(error)
);
this.logger.error(
error,
`Failed to decrypt item ${item.id} with key version ${keyInfo.version}.`
);
return false;
});
if (typeof decryptedItem === "string") {
decrypted.push({ data: decryptedItem, version: item.v });
break;
}
}
if (decryptionErrors.length === keys.length) {
await this.db.failedSyncItems.add({
itemId: item.id,
itemType: itemType,
cipher: item,
errors: decryptionErrors,
dateSynced: Date.now()
});
}
}
}
await this.mergeDecryptedItems(itemType, decrypted, options);
}
/**
* Attempt to decrypt and merge failed sync items using the provided key.
* Successfully recovered items are removed from failedSyncItems without
* soft-deleting the underlying item. Failures append to the stored error list.
*/
async retryFailedItems(
ids: string[],
key: SerializedKey
): Promise<{
succeeded: string[];
failed: { id: string; error: string }[];
}> {
const result: {
succeeded: string[];
failed: { id: string; error: string }[];
} = { succeeded: [], failed: [] };
if (ids.length <= 0) return result;
const items = await this.db.failedSyncItems.all.items(ids);
for (const failedItem of items) {
try {
await this.retryFailedItem(failedItem, key);
result.succeeded.push(failedItem.id);
} catch (error) {
const message =
error instanceof Error ? error.message : JSON.stringify(error);
this.logger.error(
error,
`Failed to retry decryption for item ${failedItem.itemId}.`
);
result.failed.push({ id: failedItem.id, error: message });
await this.db.failedSyncItems.collection.upsert({
...failedItem,
errors: [...(failedItem.errors ?? []), message],
dateModified: Date.now()
});
}
}
return result;
}
private async retryFailedItem(
failedItem: {
id: string;
itemId: string;
itemType: SyncableItemType;
cipher: SyncItem;
},
key: SerializedKey
) {
const { cipher, itemType, itemId } = failedItem;
const decrypted = await this.db.storage().decrypt(key, cipher);
const merged = await this.mergeDecryptedItems(itemType, [
{ data: decrypted, version: cipher.v }
]);
if (merged.length === 0) {
throw new Error(`Failed to deserialize item ${itemId}.`);
}
await this.db.failedSyncItems.remove([failedItem.id]);
}
/**
* Deserialize decrypted payloads, merge with local items, and persist.
* Shared by processChunk and retryFailedItems.
*/
private async mergeDecryptedItems(
itemType: SyncableItemType,
decrypted: { data: string; version: number }[],
options: Pick<SyncOptions, "offlineMode"> = {}
): Promise<(MaybeDeletedItem<Item> | undefined)[]> {
if (decrypted.length === 0) return [];
const deserialized: MaybeDeletedItem<Item>[] = [];
for (const { data, version } of decrypted) {
const decryptedItem = JSON.parse(data) as MaybeDeletedItem<Item>;
const item = await deserializeItem(
decryptedItem,
itemType,
@@ -417,38 +539,38 @@ export class Sync {
if (item) deserialized.push(item);
}
const collectionType = SYNC_COLLECTIONS_MAP[itemType];
if (deserialized.length === 0) return [];
const collectionType = SYNC_COLLECTIONS_MAP[itemType];
if (!collectionType) {
this.logger.error(
new Error(
`Unknown collection type for item type ${itemType}. Skipping chunk.`
`Unknown collection type for item type ${itemType}. Skipping items.`
)
);
return;
return [];
}
const collection = this.db[collectionType].collection;
const localItems = await collection.records(chunk.items.map((i) => i.id));
const localItems = await collection.records(deserialized.map((i) => i.id));
let items: (MaybeDeletedItem<Item> | undefined)[] = [];
if (itemType === "content") {
items = deserialized.map((item) =>
this.merger.mergeContent(item, localItems[item.id])
);
} else if (itemType === "attachment") {
items = await Promise.all(
deserialized.map((item) =>
this.merger.mergeAttachment(
item as MaybeDeletedItem<Attachment>,
localItems[item.id] as MaybeDeletedItem<Attachment>
)
)
);
} else {
items =
itemType === "attachment"
? await Promise.all(
deserialized.map((item) =>
this.merger.mergeAttachment(
item as MaybeDeletedItem<Attachment>,
localItems[item.id] as MaybeDeletedItem<Attachment>
)
)
)
: deserialized.map((item) =>
this.merger.mergeItem(item, localItems[item.id])
);
items = deserialized.map((item) =>
this.merger.mergeItem(item, localItems[item.id])
);
}
if (itemType === "note" || itemType === "content") {
@@ -473,6 +595,7 @@ export class Sync {
ids: items.map((i) => i?.id)
});
await collection.put(items as any);
return items;
}
private async pushItem(deviceId: string, item: SyncTransferItem) {

View File

@@ -0,0 +1,99 @@
/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { BaseItem, FailedSyncItem } from "../types.js";
import Database from "../api/index.js";
import { ICollection } from "./collection.js";
import { SQLCollection } from "../database/sql-collection.js";
import { isFalse } from "../database/index.js";
import { getId } from "../utils/id.js";
import { SYNC_COLLECTIONS_MAP } from "../api/sync/types.js";
import type { SyncableItemType } from "../api/sync/types.js";
import { logger } from "../logger.js";
export class FailedSyncItems implements ICollection {
name = "failedsyncitems";
readonly collection: SQLCollection<"failedsyncitems", FailedSyncItem>;
constructor(private readonly db: Database) {
this.collection = new SQLCollection(
db.sql,
db.transaction,
"failedsyncitems",
db.eventManager,
db.sanitizer
);
}
init() {
return this.collection.init();
}
async add(item: Omit<FailedSyncItem, keyof BaseItem<"failedsyncitem">>) {
const now = Date.now();
const id = getId();
await this.collection.upsert({
type: "failedsyncitem",
dateCreated: now,
dateModified: now,
...item,
id
});
return id;
}
get all() {
return this.collection.createFilter<FailedSyncItem>(
(qb) => qb.where(isFalse("deleted")),
this.db.options?.batchSize
);
}
async delete(ids: string[]) {
const groupedItems: Partial<Record<SyncableItemType, FailedSyncItem[]>> =
{};
for (const item of await this.all.items(ids)) {
const grouped = groupedItems[item.itemType] ?? [];
grouped.push(item);
groupedItems[item.itemType] = grouped;
}
for (const [itemType, items] of Object.entries(groupedItems)) {
const collectionType = SYNC_COLLECTIONS_MAP[itemType as SyncableItemType];
const collection = this.db[collectionType];
if (!collectionType || !collection) {
logger.error(
new Error(`Unknown collection type: ${itemType}`),
`Failed to delete failed sync items because the collection type is unknown.`
);
continue;
}
await collection.collection.softDelete(items.map((item) => item.itemId));
}
await this.collection.delete(ids);
}
async remove(ids: string[]) {
if (ids.length <= 0) return;
await this.collection.delete(ids);
}
async clear() {
const ids = await this.all.ids();
await this.delete(ids);
}
}

View File

@@ -41,6 +41,7 @@ import {
Attachment,
Color,
ContentItem,
FailedSyncItem,
HistorySession,
InboxItemHistory,
ItemReference,
@@ -94,6 +95,7 @@ export interface DatabaseSchema {
vaults: SQLiteItem<Vault>;
monographs: SQLiteItem<Monograph>;
inboxitemshistory: SQLiteItem<InboxItemHistory>;
failedsyncitems: SQLiteItem<FailedSyncItem>;
}
export type RawDatabaseSchema = DatabaseSchema & {
@@ -279,6 +281,10 @@ const DataMappers: Partial<Record<ItemType, (row: any) => void>> = {
},
trash: (row) => {
if (row.expiryDate) row.expiryDate = JSON.parse(row.expiryDate);
},
failedsyncitem: (row) => {
if (row.cipher) row.cipher = JSON.parse(row.cipher);
if (row.errors) row.errors = JSON.parse(row.errors);
}
};

View File

@@ -496,6 +496,21 @@ export class NNMigrationProvider implements MigrationProvider {
.addColumn("errorContext", "text")
.execute();
}
},
"a-2026-08-10": {
async up(db) {
await db.schema
.createTable("failedsyncitems")
.ifNotExists()
.modifyEnd(sql`without rowid`)
.$call(addBaseColumns)
.addColumn("dateSynced", "integer")
.addColumn("cipher", "text")
.addColumn("errors", "text")
.addColumn("itemType", "text")
.addColumn("itemId", "text")
.execute();
}
}
};
}

View File

@@ -703,7 +703,8 @@ const VALID_SORT_OPTIONS: Record<
shortcuts: [],
vaults: [],
monographs: [],
inboxitemshistory: []
inboxitemshistory: [],
failedsyncitems: []
};
function sanitizeSortOptions(type: keyof DatabaseSchema, options: SortOptions) {

View File

@@ -19,6 +19,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import { Cipher } from "@notesnook/crypto";
import { isCipher } from "./utils/index.js";
import type { SyncableItemType, SyncItem } from "./api/sync/types.js";
export type TimeFormat = "12-hour" | "24-hour";
export type DayFormat = "short" | "long";
@@ -85,6 +86,7 @@ export type Collections = {
vaults: "vault";
monographs: "monograph";
inboxitemshistory: "inboxitemhistory";
failedsyncitems: "failedsyncitem";
/**
* @deprecated only kept here for migration purposes
@@ -117,6 +119,7 @@ export type GroupableItem = ValueOf<
| "vault"
| "monograph"
| "inboxitemhistory"
| "failedsyncitem"
>
>;
@@ -140,6 +143,7 @@ export type ItemMap = {
searchResult: HighlightedResult;
monograph: Monograph;
inboxitemhistory: InboxItemHistory;
failedsyncitem: FailedSyncItem;
/**
* @deprecated only kept here for migration purposes
@@ -547,6 +551,14 @@ export interface InboxItemHistory extends BaseItem<"inboxitemhistory"> {
errorContext?: string;
}
export interface FailedSyncItem extends BaseItem<"failedsyncitem"> {
itemType: SyncableItemType;
itemId: string;
dateSynced: number;
cipher: SyncItem;
errors: string[];
}
export type Match = {
prefix: string;
match: string;

View File

@@ -512,6 +512,10 @@ msgstr "{count, plural, one {Version deleted} other {# versions deleted}}"
msgid "{count} characters"
msgstr "{count} characters"
#: src/strings.ts:2823
msgid "{count} item(s) decrypted successfully"
msgstr "{count} item(s) decrypted successfully"
#: src/strings.ts:1566
msgid "{days, plural, one {1 day} other {# days}}"
msgstr "{days, plural, one {1 day} other {# days}}"
@@ -596,6 +600,10 @@ msgstr "{platform, select, android {Backup file saved in \"Notesnook backups\" f
msgid "{selected} selected"
msgstr "{selected} selected"
#: src/strings.ts:2826
msgid "{succeeded} item(s) recovered, {failed} still failed"
msgstr "{succeeded} item(s) recovered, {failed} still failed"
#: src/strings.ts:382
msgid "{type, select, github {v{version} has been released on GitHub} other {v{version} has been released}}"
msgstr "{type, select, github {v{version} has been released on GitHub} other {v{version} has been released}}"
@@ -995,6 +1003,10 @@ msgstr "Are you sure you want to clear trash?"
msgid "Are you sure you want to delete all failed inbox items?"
msgstr "Are you sure you want to delete all failed inbox items?"
#: src/strings.ts:2819
msgid "Are you sure you want to delete all failed sync items? This will also remove the corresponding local items."
msgstr "Are you sure you want to delete all failed sync items? This will also remove the corresponding local items."
#: src/strings.ts:2734
msgid "Are you sure you want to delete this attachment?"
msgstr "Are you sure you want to delete this attachment?"
@@ -2258,6 +2270,10 @@ msgstr "Debugging"
msgid "Decrease {title}"
msgstr "Decrease {title}"
#: src/strings.ts:2824
msgid "Decryption failed"
msgstr "Decryption failed"
#: src/strings.ts:660
#: src/strings.ts:2010
msgid "Default"
@@ -3027,6 +3043,10 @@ msgstr "Failed"
msgid "Failed inbox items"
msgstr "Failed inbox items"
#: src/strings.ts:2814
msgid "Failed sync items"
msgstr "Failed sync items"
#: src/strings.ts:2653
msgid "Failed to attach file"
msgstr "Failed to attach file"
@@ -3792,6 +3812,10 @@ msgstr "Item"
msgid "Item deleted"
msgstr "Item deleted"
#: src/strings.ts:2827
msgid "Item ID"
msgstr "Item ID"
#: src/strings.ts:311
#: src/strings.ts:1705
msgid "items"
@@ -4500,6 +4524,10 @@ msgstr "No encryption key found"
msgid "No failed inbox items"
msgstr "No failed inbox items"
#: src/strings.ts:2817
msgid "No failed sync items"
msgstr "No failed sync items"
#: src/strings.ts:1667
msgid "No headings found"
msgstr "No headings found"
@@ -5846,6 +5874,14 @@ msgstr "Resubscribe to Pro"
msgid "Retry"
msgstr "Retry"
#: src/strings.ts:2821
msgid "Retry with current keys"
msgstr "Retry with current keys"
#: src/strings.ts:2820
msgid "Retry with custom key"
msgstr "Retry with custom key"
#: src/strings.ts:513
msgid "Reupload"
msgstr "Reupload"
@@ -7127,6 +7163,10 @@ msgstr "Trash gets automatically cleaned up after {days} days"
msgid "Trash gets automatically cleaned up daily"
msgstr "Trash gets automatically cleaned up daily"
#: src/strings.ts:2813
msgid "Troubleshoot"
msgstr "Troubleshoot"
#: src/strings.ts:2553
msgid "Try {plan} for free"
msgstr "Try {plan} for free"
@@ -7593,6 +7633,10 @@ msgstr "View and share debug logs"
msgid "View failed inbox items and error contexts"
msgstr "View failed inbox items and error contexts"
#: src/strings.ts:2816
msgid "View items that failed to decrypt during sync and retry them with a different key"
msgstr "View items that failed to decrypt during sync and retry them with a different key"
#: src/strings.ts:1749
msgid "View receipt"
msgstr "View receipt"

View File

@@ -512,6 +512,10 @@ msgstr ""
msgid "{count} characters"
msgstr ""
#: src/strings.ts:2823
msgid "{count} item(s) decrypted successfully"
msgstr ""
#: src/strings.ts:1566
msgid "{days, plural, one {1 day} other {# days}}"
msgstr ""
@@ -596,6 +600,10 @@ msgstr ""
msgid "{selected} selected"
msgstr ""
#: src/strings.ts:2826
msgid "{succeeded} item(s) recovered, {failed} still failed"
msgstr ""
#: src/strings.ts:382
msgid "{type, select, github {v{version} has been released on GitHub} other {v{version} has been released}}"
msgstr ""
@@ -995,6 +1003,10 @@ msgstr ""
msgid "Are you sure you want to delete all failed inbox items?"
msgstr ""
#: src/strings.ts:2819
msgid "Are you sure you want to delete all failed sync items? This will also remove the corresponding local items."
msgstr ""
#: src/strings.ts:2734
msgid "Are you sure you want to delete this attachment?"
msgstr ""
@@ -2247,6 +2259,10 @@ msgstr ""
msgid "Decrease {title}"
msgstr ""
#: src/strings.ts:2824
msgid "Decryption failed"
msgstr ""
#: src/strings.ts:660
#: src/strings.ts:2010
msgid "Default"
@@ -3016,6 +3032,10 @@ msgstr ""
msgid "Failed inbox items"
msgstr ""
#: src/strings.ts:2814
msgid "Failed sync items"
msgstr ""
#: src/strings.ts:2653
msgid "Failed to attach file"
msgstr ""
@@ -3772,6 +3792,10 @@ msgstr ""
msgid "Item deleted"
msgstr ""
#: src/strings.ts:2827
msgid "Item ID"
msgstr ""
#: src/strings.ts:311
#: src/strings.ts:1705
msgid "items"
@@ -4480,6 +4504,10 @@ msgstr ""
msgid "No failed inbox items"
msgstr ""
#: src/strings.ts:2817
msgid "No failed sync items"
msgstr ""
#: src/strings.ts:1667
msgid "No headings found"
msgstr ""
@@ -5820,6 +5848,14 @@ msgstr ""
msgid "Retry"
msgstr ""
#: src/strings.ts:2821
msgid "Retry with current keys"
msgstr ""
#: src/strings.ts:2820
msgid "Retry with custom key"
msgstr ""
#: src/strings.ts:513
msgid "Reupload"
msgstr ""
@@ -7086,6 +7122,10 @@ msgstr ""
msgid "Trash gets automatically cleaned up daily"
msgstr ""
#: src/strings.ts:2813
msgid "Troubleshoot"
msgstr ""
#: src/strings.ts:2553
msgid "Try {plan} for free"
msgstr ""
@@ -7543,6 +7583,10 @@ msgstr ""
msgid "View failed inbox items and error contexts"
msgstr ""
#: src/strings.ts:2816
msgid "View items that failed to decrypt during sync and retry them with a different key"
msgstr ""
#: src/strings.ts:1749
msgid "View receipt"
msgstr ""

View File

@@ -2809,5 +2809,20 @@ Continue without attachments?`,
versionDeleted: () => actions.deleted.version(1),
offlineMode: () => t`Offline mode`,
offlineModeDesc: () =>
t`Using Notesnook without an account will **NOT** sync your notes across devices and could result in data loss if you lose access to your device or uninstall the app. Make sure to backup your notes regularly.`
t`Using Notesnook without an account will **NOT** sync your notes across devices and could result in data loss if you lose access to your device or uninstall the app. Make sure to backup your notes regularly.`,
troubleshoot: () => t`Troubleshoot`,
failedSyncItems: () => t`Failed sync items`,
failedSyncItemsDesc: () =>
t`View items that failed to decrypt during sync and retry them with a different key`,
noFailedSyncItems: () => t`No failed sync items`,
deleteAllFailedSyncItemsDesc: () =>
t`Are you sure you want to delete all failed sync items? This will also remove the corresponding local items.`,
retryWithCustomKey: () => t`Retry with custom key`,
retryWithCurrentKeys: () => t`Retry with current keys`,
decryptionSucceeded: (count: number) =>
t`${count} item(s) decrypted successfully`,
decryptionFailed: () => t`Decryption failed`,
decryptionPartialSuccess: (succeeded: number, failed: number) =>
t`${succeeded} item(s) recovered, ${failed} still failed`,
itemId: () => t`Item ID`
};