refac: kb sync

backend/open_webui/routers/files.py

@@ -1,4 +1,5 @@
 import asyncio
+import hashlib
 import json
 import logging
 import os
@@ -290,6 +291,10 @@ async def upload_file_handler(
             },
         )
 
+        # SHA-256 of raw uploaded bytes for incremental sync diffing.
+        # If the client pre-computed and sent file_hash, use that.
+        file_hash = file_metadata.get('file_hash') or hashlib.sha256(contents).hexdigest()
+
         file_item = await Files.insert_new_file(
             user.id,
             FileForm(
@@ -304,6 +309,7 @@ async def upload_file_handler(
                         'name': name,
                         'content_type': (file.content_type if isinstance(file.content_type, str) else None),
                         'size': len(contents),
+                        'file_hash': file_hash,
                         'data': file_metadata,
                     },
                 }

backend/open_webui/routers/knowledge.py

@@ -984,6 +984,184 @@ async def reset_knowledge_by_id(
     return knowledge
 
 
+############################
+# SyncKnowledgeDiff
+############################
+
+
+class FileManifestEntry(BaseModel):
+    filename: str       # basename: "readme.md"
+    path: str           # relative dir: "docs/api" or "" for root
+    checksum: str       # SHA-256 of raw bytes
+    size: int
+
+
+class SyncDiffForm(BaseModel):
+    manifest: list[FileManifestEntry]
+
+
+class SyncDiffResponse(BaseModel):
+    added: list[dict]               # [{filename, path}] — new files
+    modified: list[dict]            # [{filename, path, stale_file_id}] — changed files
+    deleted: list[dict]             # [{file_id, filename}] — files to remove
+    mkdir: list[str]                # directory paths to create
+    rmdir: list[str]                # directory IDs to remove
+    unmodified_count: int
+
+
+@router.post('/{id}/sync/diff', response_model=SyncDiffResponse)
+async def sync_knowledge_diff(
+    id: str,
+    form_data: SyncDiffForm,
+    user=Depends(get_verified_user),
+    db: AsyncSession = Depends(get_async_session),
+):
+    """
+    Compare a local file manifest against the knowledge base to determine
+    which files need uploading, removing, and which directories to create/remove.
+    """
+    await _verify_knowledge_write_access(id, user, db)
+
+    # ── Index existing state ──
+    knowledge_files = await Knowledges.get_files_with_directory_ids(id, db=db)
+    existing_directories = await Knowledges.get_all_directories(id, db=db)
+
+    # Build directory path lookups
+    directory_path_by_id: dict[str, str] = {}
+    directory_id_by_path: dict[str, str] = {}
+    for directory in existing_directories:
+        segments = [directory.name]
+        parent_id = directory.parent_id
+        while parent_id:
+            parent = next((d for d in existing_directories if d.id == parent_id), None)
+            if not parent:
+                break
+            segments.insert(0, parent.name)
+            parent_id = parent.parent_id
+        full_path = '/'.join(segments)
+        directory_path_by_id[directory.id] = full_path
+        directory_id_by_path[full_path] = directory.id
+
+    # Index existing files by (path, filename) → {file_id, checksum}
+    indexed_files: dict[tuple[str, str], dict] = {}
+    for file_model, directory_id in knowledge_files:
+        file_path = directory_path_by_id.get(directory_id, '') if directory_id else ''
+        stored_checksum = (file_model.meta or {}).get('file_hash')
+        indexed_files[(file_path, file_model.filename)] = {
+            'file_id': file_model.id,
+            'checksum': stored_checksum,
+        }
+
+    # ── Diff files ──
+    added: list[dict] = []
+    modified: list[dict] = []
+    deleted: list[dict] = []
+    unmodified_count = 0
+    manifest_keys: set[tuple[str, str]] = set()
+
+    for entry in form_data.manifest:
+        key = (entry.path, entry.filename)
+        manifest_keys.add(key)
+
+        if key not in indexed_files:
+            added.append({'filename': entry.filename, 'path': entry.path})
+        elif indexed_files[key]['checksum'] != entry.checksum:
+            modified.append({
+                'filename': entry.filename,
+                'path': entry.path,
+                'stale_file_id': indexed_files[key]['file_id'],
+            })
+        else:
+            unmodified_count += 1
+
+    for key, file_info in indexed_files.items():
+        if key not in manifest_keys:
+            deleted.append({'file_id': file_info['file_id'], 'filename': key[1]})
+
+    # ── Diff directories ──
+    required_directory_paths: set[str] = set()
+    for entry in form_data.manifest:
+        if entry.path:
+            segments = entry.path.split('/')
+            for depth in range(len(segments)):
+                required_directory_paths.add('/'.join(segments[:depth + 1]))
+
+    mkdir = sorted(
+        [p for p in required_directory_paths if p not in directory_id_by_path],
+        key=lambda p: p.count('/')
+    )
+
+    orphaned_directory_paths = set(directory_id_by_path) - required_directory_paths
+    rmdir = [directory_id_by_path[p] for p in orphaned_directory_paths]
+
+    return SyncDiffResponse(
+        added=added,
+        modified=modified,
+        deleted=deleted,
+        mkdir=mkdir,
+        rmdir=rmdir,
+        unmodified_count=unmodified_count,
+    )
+
+
+############################
+# SyncKnowledgeCleanup
+############################
+
+
+class SyncCleanupForm(BaseModel):
+    file_ids: list[str]         # file IDs to delete
+    dir_ids: list[str] = []     # directory IDs to rmdir
+
+
+@router.post('/{id}/sync/cleanup')
+async def sync_knowledge_cleanup(
+    id: str,
+    form_data: SyncCleanupForm,
+    user=Depends(get_verified_user),
+    db: AsyncSession = Depends(get_async_session),
+):
+    """
+    Remove stale files and orphaned directories from a knowledge base
+    after an incremental sync.
+    """
+    await _verify_knowledge_write_access(id, user, db)
+
+    # ── Remove deleted files ──
+    for file_id in form_data.file_ids:
+        file = await Files.get_file_by_id(file_id, db=db)
+        if not file:
+            continue
+
+        await Knowledges.remove_file_from_knowledge_by_id(id, file_id, db=db)
+
+        try:
+            await ASYNC_VECTOR_DB_CLIENT.delete(
+                collection_name=id, filter={'file_id': file_id}
+            )
+            await ASYNC_VECTOR_DB_CLIENT.delete(
+                collection_name=id, filter={'hash': file.hash}
+            )
+        except Exception:
+            pass
+
+        try:
+            collection_name = f'file-{file_id}'
+            if await ASYNC_VECTOR_DB_CLIENT.has_collection(collection_name):
+                await ASYNC_VECTOR_DB_CLIENT.delete_collection(collection_name)
+        except Exception:
+            pass
+
+        if file.user_id == user.id or user.role == 'admin':
+            await Files.delete_file_by_id(file_id, db=db)
+
+    # ── Remove orphaned directories (children before parents) ──
+    for dir_id in reversed(form_data.dir_ids):
+        await Knowledges.delete_directory(dir_id, move_files_to_parent=False, db=db)
+
+    return {'status': True}
+
+
 ############################
 # AddFilesToKnowledge
 ############################

src/lib/apis/knowledge/index.ts

@@ -470,9 +470,83 @@ export const resetKnowledgeById = async (token: string, id: string) => {
 	return res;
 };
 
+export const syncKnowledgeDiff = async (
+	token: string,
+	id: string,
+	manifest: Array<{ filename: string; path: string; checksum: string; size: number }>
+) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/${id}/sync/diff`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({ manifest })
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.then((json) => {
+			return json;
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.error(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const syncKnowledgeCleanup = async (
+	token: string,
+	id: string,
+	fileIds: string[],
+	dirIds: string[] = []
+) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/${id}/sync/cleanup`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({ file_ids: fileIds, dir_ids: dirIds })
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.then((json) => {
+			return json;
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.error(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
 export const deleteKnowledgeById = async (token: string, id: string) => {
 	let error = null;
 
+
 	const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/${id}/delete`, {
 		method: 'DELETE',
 		headers: {

src/lib/components/workspace/Knowledge/KnowledgeBase.svelte

@@ -37,11 +37,14 @@
 		createKnowledgeDirectory,
 		updateKnowledgeDirectory,
 		deleteKnowledgeDirectory,
-		moveFileInKnowledge
+		moveFileInKnowledge,
+		syncKnowledgeDiff,
+		syncKnowledgeCleanup
 	} from '$lib/apis/knowledge';
 	import { processWeb, processYoutubeVideo } from '$lib/apis/retrieval';
 
 	import { blobToFile, isYoutubeUrl, copyToClipboard } from '$lib/utils';
+	import { computeFileHash } from '$lib/utils/hash';
 
 	import Spinner from '$lib/components/common/Spinner.svelte';
 	import Tooltip from '$lib/components/common/Tooltip.svelte';
@@ -75,6 +78,7 @@
 	let showNewDirectoryModal = false;
 
 	let showSyncConfirmModal = false;
+	let pendingSyncFiles: Array<{path: string, filename: string, file: File}> | null = null;
 	let showAccessControlModal = false;
 
 	let minSize = 0;
@@ -537,25 +541,171 @@
 		}
 	};
 
-	// Helper function to maintain file paths within zip
-	const syncDirectoryHandler = async () => {
-		if (fileItems.length > 0) {
-			const res = await resetKnowledgeById(localStorage.token, id).catch((e) => {
-				toast.error(`${e}`);
-			});
+	// Collect files from a directory without uploading — returns {path, filename, file}[]
+	const collectDirectoryFiles = async (): Promise<Array<{path: string, filename: string, file: File}> | null> => {
+		const isFileSystemAccessSupported = 'showDirectoryPicker' in window;
 
-			if (res) {
-				fileItems = [];
-				toast.success($i18n.t('Knowledge reset successfully.'));
+		try {
+			if (isFileSystemAccessSupported) {
+				const dirHandle = await window.showDirectoryPicker();
+				const collected: Array<{path: string, filename: string, file: File}> = [];
 
-				// Upload directory
-				uploadDirectoryHandler();
+				async function traverse(handle: FileSystemDirectoryHandle, dirPath = '') {
+					for await (const entry of handle.values()) {
+						if (entry.name.startsWith('.')) continue;
+						const entryPath = dirPath ? `${dirPath}/${entry.name}` : entry.name;
+						if (hasHiddenFolder(entryPath)) continue;
+
+						if (entry.kind === 'file') {
+							const file = await entry.getFile();
+							collected.push({ path: dirPath, filename: entry.name, file });
+						} else if (entry.kind === 'directory') {
+							await traverse(entry, entryPath);
+						}
+					}
+				}
+
+				await traverse(dirHandle);
+				return collected;
+			} else {
+				// Firefox fallback
+				return new Promise((resolve, reject) => {
+					const input = document.createElement('input');
+					input.type = 'file';
+					input.webkitdirectory = true;
+					input.directory = true;
+					input.multiple = true;
+					input.style.display = 'none';
+					document.body.appendChild(input);
+
+					input.onchange = () => {
+						try {
+							const files = Array.from(input.files || [])
+								.filter((file) => !hasHiddenFolder(file.webkitRelativePath) && !file.name.startsWith('.'));
+
+							const collected = files.map((file) => {
+								const parts = file.webkitRelativePath.split('/');
+								// Remove root dir name, extract path and filename
+								const withoutRoot = parts.slice(1);
+								const filename = withoutRoot.pop() || file.name;
+								const path = withoutRoot.join('/');
+								return { path, filename, file };
+							});
+
+							document.body.removeChild(input);
+							resolve(collected);
+						} catch (error) {
+							document.body.removeChild(input);
+							reject(error);
+						}
+					};
+
+					input.onerror = (error) => {
+						document.body.removeChild(input);
+						reject(error);
+					};
+
+					input.click();
+				});
 			}
-		} else {
-			uploadDirectoryHandler();
+		} catch (error) {
+			handleUploadError(error);
+			return null;
 		}
 	};
 
+	// Incremental sync: hash locally → diff on server → upload only what changed
+	const syncDirectoryHandler = async () => {
+		if (!pendingSyncFiles?.length) return;
+
+		// ── 2. Compute checksums ──
+		toast.info($i18n.t('Computing checksums...'));
+		const manifest = await Promise.all(
+			pendingSyncFiles.map(async (entry) => ({
+				...entry,
+				checksum: await computeFileHash(entry.file),
+				size: entry.file.size
+			}))
+		);
+		pendingSyncFiles = null;
+
+		// ── 3. Diff against knowledge base ──
+		toast.info($i18n.t('Comparing with knowledge base...'));
+		const diff = await syncKnowledgeDiff(
+			localStorage.token,
+			id,
+			manifest.map(({ filename, path, checksum, size }) => ({ filename, path, checksum, size }))
+		);
+
+		if (!diff) {
+			toast.error($i18n.t('Failed to compare files.'));
+			return;
+		}
+
+		// ── 4. mkdir — create missing directories (parents first) ──
+		const createdDirectoryIds: Record<string, string> = {};
+		for (const dirPath of diff.mkdir) {
+			const segments = dirPath.split('/');
+			const name = segments.at(-1)!;
+			const parentPath = segments.slice(0, -1).join('/');
+			const parentId = parentPath ? createdDirectoryIds[parentPath] : null;
+
+			const directory = await createKnowledgeDirectory(
+				localStorage.token, knowledge.id, name, parentId
+			);
+			if (directory) {
+				createdDirectoryIds[dirPath] = directory.id;
+			}
+		}
+
+		// ── 5. Upload added + modified files ──
+		const filesToUpload = manifest.filter((entry) =>
+			diff.added.some((a: any) => a.filename === entry.filename && a.path === entry.path) ||
+			diff.modified.some((m: any) => m.filename === entry.filename && m.path === entry.path)
+		);
+
+		let uploadedCount = 0;
+		for (const entry of filesToUpload) {
+			const fileObject = new File([entry.file], entry.filename, { type: entry.file.type });
+			await uploadFile(localStorage.token, fileObject, {
+				knowledge_id: knowledge.id,
+				file_hash: entry.checksum,
+				directory_id: entry.path ? createdDirectoryIds[entry.path] : null
+			});
+			uploadedCount++;
+			toast.info(
+				$i18n.t('Uploading: {{current}}/{{total}}', {
+					current: uploadedCount,
+					total: filesToUpload.length
+				})
+			);
+		}
+
+		// ── 6. Cleanup — remove deleted files + rmdir orphaned directories ──
+		const staleFileIds = [
+			...diff.deleted.map((d: any) => d.file_id),
+			...diff.modified.map((m: any) => m.stale_file_id)
+		];
+
+		if (staleFileIds.length > 0 || diff.rmdir.length > 0) {
+			await syncKnowledgeCleanup(localStorage.token, id, staleFileIds, diff.rmdir);
+		}
+
+		// ── 7. Report ──
+		toast.success(
+			$i18n.t(
+				'Sync complete: {{added}} added, {{modified}} modified, {{deleted}} deleted, {{unmodified}} unmodified',
+				{
+					added: diff.added.length,
+					modified: diff.modified.length,
+					deleted: diff.deleted.length,
+					unmodified: diff.unmodified_count
+				}
+			)
+		);
+		init();
+	};
+
 	const addFileHandler = async (fileId) => {
 		const res = await addFileToKnowledgeById(
 			localStorage.token,
@@ -924,11 +1074,15 @@
 <SyncConfirmDialog
 	bind:show={showSyncConfirmModal}
 	message={$i18n.t(
-		'This will reset the knowledge base and sync all files. Do you wish to continue?'
+		'{{count}} files selected. Only new and modified files will be uploaded. Deleted files will be removed. The folder structure will be mirrored. Continue?',
+		{ count: pendingSyncFiles?.length ?? 0 }
 	)}
 	on:confirm={() => {
 		syncDirectoryHandler();
 	}}
+	on:cancel={() => {
+		pendingSyncFiles = null;
+	}}
 />
 
 <AttachWebpageModal
@@ -1113,8 +1267,11 @@
 										document.getElementById('files-input').click();
 									}
 								}}
-								onSync={() => {
-									showSyncConfirmModal = true;
+								onSync={async () => {
+									pendingSyncFiles = await collectDirectoryFiles();
+									if (pendingSyncFiles?.length) {
+										showSyncConfirmModal = true;
+									}
 								}}
 							/>
 						</div>

src/lib/components/workspace/Knowledge/KnowledgeBase/AddContentMenu.svelte

@@ -90,7 +90,7 @@
 
 			<Tooltip
 				content={$i18n.t(
-					'This option will delete all existing files in the collection and replace them with newly uploaded files.'
+					'Sync a local directory with this knowledge base. Only new and modified files will be uploaded. The directory structure will be mirrored.'
 				)}
 				className="w-full"
 			>

src/lib/utils/hash.ts

@@ -0,0 +1,115 @@
+/**
+ * Compute SHA-256 checksum of a File.
+ *
+ * Uses the Web Crypto API when available (HTTPS / localhost).
+ * Falls back to a pure-JS implementation for plain HTTP deployments
+ * where crypto.subtle is unavailable.
+ *
+ * @returns lowercase hex-encoded SHA-256 digest
+ */
+export async function computeFileHash(file: File): Promise<string> {
+	const buffer = await file.arrayBuffer();
+
+	if (globalThis.crypto?.subtle) {
+		const digest = await crypto.subtle.digest('SHA-256', buffer);
+		return hexEncode(new Uint8Array(digest));
+	}
+
+	return sha256Fallback(new Uint8Array(buffer));
+}
+
+function hexEncode(bytes: Uint8Array): string {
+	const hex: string[] = new Array(bytes.length);
+	for (let i = 0; i < bytes.length; i++) {
+		hex[i] = bytes[i].toString(16).padStart(2, '0');
+	}
+	return hex.join('');
+}
+
+// ── Pure-JS SHA-256 (RFC 6234) ──────────────────────────────────────────────
+// Used only when crypto.subtle is unavailable (HTTP deployments).
+
+const K: Uint32Array = new Uint32Array([
+	0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4,
+	0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe,
+	0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f,
+	0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
+	0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc,
+	0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b,
+	0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116,
+	0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
+	0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7,
+	0xc67178f2
+]);
+
+function rotr(n: number, x: number): number {
+	return (x >>> n) | (x << (32 - n));
+}
+
+function sha256Fallback(data: Uint8Array): string {
+	// Pre-processing: padding
+	const bitLen = data.length * 8;
+	const padLen = (((data.length + 8) >>> 6) + 1) << 6;
+	const padded = new Uint8Array(padLen);
+	padded.set(data);
+	padded[data.length] = 0x80;
+
+	// Append original length in bits as 64-bit big-endian
+	const view = new DataView(padded.buffer);
+	view.setUint32(padLen - 4, bitLen, false);
+
+	// Initial hash values
+	let h0 = 0x6a09e667;
+	let h1 = 0xbb67ae85;
+	let h2 = 0x3c6ef372;
+	let h3 = 0xa54ff53a;
+	let h4 = 0x510e527f;
+	let h5 = 0x9b05688c;
+	let h6 = 0x1f83d9ab;
+	let h7 = 0x5be0cd19;
+
+	const w = new Uint32Array(64);
+
+	for (let offset = 0; offset < padLen; offset += 64) {
+		// Prepare message schedule
+		for (let i = 0; i < 16; i++) {
+			w[i] = view.getUint32(offset + i * 4, false);
+		}
+		for (let i = 16; i < 64; i++) {
+			const s0 = rotr(7, w[i - 15]) ^ rotr(18, w[i - 15]) ^ (w[i - 15] >>> 3);
+			const s1 = rotr(17, w[i - 2]) ^ rotr(19, w[i - 2]) ^ (w[i - 2] >>> 10);
+			w[i] = (w[i - 16] + s0 + w[i - 7] + s1) | 0;
+		}
+
+		let a = h0, b = h1, c = h2, d = h3;
+		let e = h4, f = h5, g = h6, h = h7;
+
+		for (let i = 0; i < 64; i++) {
+			const S1 = rotr(6, e) ^ rotr(11, e) ^ rotr(25, e);
+			const ch = (e & f) ^ (~e & g);
+			const temp1 = (h + S1 + ch + K[i] + w[i]) | 0;
+			const S0 = rotr(2, a) ^ rotr(13, a) ^ rotr(22, a);
+			const maj = (a & b) ^ (a & c) ^ (b & c);
+			const temp2 = (S0 + maj) | 0;
+
+			h = g; g = f; f = e;
+			e = (d + temp1) | 0;
+			d = c; c = b; b = a;
+			a = (temp1 + temp2) | 0;
+		}
+
+		h0 = (h0 + a) | 0; h1 = (h1 + b) | 0;
+		h2 = (h2 + c) | 0; h3 = (h3 + d) | 0;
+		h4 = (h4 + e) | 0; h5 = (h5 + f) | 0;
+		h6 = (h6 + g) | 0; h7 = (h7 + h) | 0;
+	}
+
+	const result = new Uint8Array(32);
+	const out = new DataView(result.buffer);
+	out.setUint32(0, h0, false); out.setUint32(4, h1, false);
+	out.setUint32(8, h2, false); out.setUint32(12, h3, false);
+	out.setUint32(16, h4, false); out.setUint32(20, h5, false);
+	out.setUint32(24, h6, false); out.setUint32(28, h7, false);
+
+	return hexEncode(result);
+}