diff --git a/backend/open_webui/routers/files.py b/backend/open_webui/routers/files.py index 422bad965d..8a09601b18 100644 --- a/backend/open_webui/routers/files.py +++ b/backend/open_webui/routers/files.py @@ -1,4 +1,5 @@ import asyncio +import hashlib import json import logging import os @@ -290,6 +291,10 @@ async def upload_file_handler( }, ) + # SHA-256 of raw uploaded bytes for incremental sync diffing. + # If the client pre-computed and sent file_hash, use that. + file_hash = file_metadata.get('file_hash') or hashlib.sha256(contents).hexdigest() + file_item = await Files.insert_new_file( user.id, FileForm( @@ -304,6 +309,7 @@ async def upload_file_handler( 'name': name, 'content_type': (file.content_type if isinstance(file.content_type, str) else None), 'size': len(contents), + 'file_hash': file_hash, 'data': file_metadata, }, } diff --git a/backend/open_webui/routers/knowledge.py b/backend/open_webui/routers/knowledge.py index ccf99bde58..69f0ae33e3 100644 --- a/backend/open_webui/routers/knowledge.py +++ b/backend/open_webui/routers/knowledge.py @@ -984,6 +984,184 @@ async def reset_knowledge_by_id( return knowledge +############################ +# SyncKnowledgeDiff +############################ + + +class FileManifestEntry(BaseModel): + filename: str # basename: "readme.md" + path: str # relative dir: "docs/api" or "" for root + checksum: str # SHA-256 of raw bytes + size: int + + +class SyncDiffForm(BaseModel): + manifest: list[FileManifestEntry] + + +class SyncDiffResponse(BaseModel): + added: list[dict] # [{filename, path}] — new files + modified: list[dict] # [{filename, path, stale_file_id}] — changed files + deleted: list[dict] # [{file_id, filename}] — files to remove + mkdir: list[str] # directory paths to create + rmdir: list[str] # directory IDs to remove + unmodified_count: int + + +@router.post('/{id}/sync/diff', response_model=SyncDiffResponse) +async def sync_knowledge_diff( + id: str, + form_data: SyncDiffForm, + user=Depends(get_verified_user), + db: AsyncSession = Depends(get_async_session), +): + """ + Compare a local file manifest against the knowledge base to determine + which files need uploading, removing, and which directories to create/remove. + """ + await _verify_knowledge_write_access(id, user, db) + + # ── Index existing state ── + knowledge_files = await Knowledges.get_files_with_directory_ids(id, db=db) + existing_directories = await Knowledges.get_all_directories(id, db=db) + + # Build directory path lookups + directory_path_by_id: dict[str, str] = {} + directory_id_by_path: dict[str, str] = {} + for directory in existing_directories: + segments = [directory.name] + parent_id = directory.parent_id + while parent_id: + parent = next((d for d in existing_directories if d.id == parent_id), None) + if not parent: + break + segments.insert(0, parent.name) + parent_id = parent.parent_id + full_path = '/'.join(segments) + directory_path_by_id[directory.id] = full_path + directory_id_by_path[full_path] = directory.id + + # Index existing files by (path, filename) → {file_id, checksum} + indexed_files: dict[tuple[str, str], dict] = {} + for file_model, directory_id in knowledge_files: + file_path = directory_path_by_id.get(directory_id, '') if directory_id else '' + stored_checksum = (file_model.meta or {}).get('file_hash') + indexed_files[(file_path, file_model.filename)] = { + 'file_id': file_model.id, + 'checksum': stored_checksum, + } + + # ── Diff files ── + added: list[dict] = [] + modified: list[dict] = [] + deleted: list[dict] = [] + unmodified_count = 0 + manifest_keys: set[tuple[str, str]] = set() + + for entry in form_data.manifest: + key = (entry.path, entry.filename) + manifest_keys.add(key) + + if key not in indexed_files: + added.append({'filename': entry.filename, 'path': entry.path}) + elif indexed_files[key]['checksum'] != entry.checksum: + modified.append({ + 'filename': entry.filename, + 'path': entry.path, + 'stale_file_id': indexed_files[key]['file_id'], + }) + else: + unmodified_count += 1 + + for key, file_info in indexed_files.items(): + if key not in manifest_keys: + deleted.append({'file_id': file_info['file_id'], 'filename': key[1]}) + + # ── Diff directories ── + required_directory_paths: set[str] = set() + for entry in form_data.manifest: + if entry.path: + segments = entry.path.split('/') + for depth in range(len(segments)): + required_directory_paths.add('/'.join(segments[:depth + 1])) + + mkdir = sorted( + [p for p in required_directory_paths if p not in directory_id_by_path], + key=lambda p: p.count('/') + ) + + orphaned_directory_paths = set(directory_id_by_path) - required_directory_paths + rmdir = [directory_id_by_path[p] for p in orphaned_directory_paths] + + return SyncDiffResponse( + added=added, + modified=modified, + deleted=deleted, + mkdir=mkdir, + rmdir=rmdir, + unmodified_count=unmodified_count, + ) + + +############################ +# SyncKnowledgeCleanup +############################ + + +class SyncCleanupForm(BaseModel): + file_ids: list[str] # file IDs to delete + dir_ids: list[str] = [] # directory IDs to rmdir + + +@router.post('/{id}/sync/cleanup') +async def sync_knowledge_cleanup( + id: str, + form_data: SyncCleanupForm, + user=Depends(get_verified_user), + db: AsyncSession = Depends(get_async_session), +): + """ + Remove stale files and orphaned directories from a knowledge base + after an incremental sync. + """ + await _verify_knowledge_write_access(id, user, db) + + # ── Remove deleted files ── + for file_id in form_data.file_ids: + file = await Files.get_file_by_id(file_id, db=db) + if not file: + continue + + await Knowledges.remove_file_from_knowledge_by_id(id, file_id, db=db) + + try: + await ASYNC_VECTOR_DB_CLIENT.delete( + collection_name=id, filter={'file_id': file_id} + ) + await ASYNC_VECTOR_DB_CLIENT.delete( + collection_name=id, filter={'hash': file.hash} + ) + except Exception: + pass + + try: + collection_name = f'file-{file_id}' + if await ASYNC_VECTOR_DB_CLIENT.has_collection(collection_name): + await ASYNC_VECTOR_DB_CLIENT.delete_collection(collection_name) + except Exception: + pass + + if file.user_id == user.id or user.role == 'admin': + await Files.delete_file_by_id(file_id, db=db) + + # ── Remove orphaned directories (children before parents) ── + for dir_id in reversed(form_data.dir_ids): + await Knowledges.delete_directory(dir_id, move_files_to_parent=False, db=db) + + return {'status': True} + + ############################ # AddFilesToKnowledge ############################ diff --git a/src/lib/apis/knowledge/index.ts b/src/lib/apis/knowledge/index.ts index 85e5c6e844..f0c1750fd4 100644 --- a/src/lib/apis/knowledge/index.ts +++ b/src/lib/apis/knowledge/index.ts @@ -470,9 +470,83 @@ export const resetKnowledgeById = async (token: string, id: string) => { return res; }; +export const syncKnowledgeDiff = async ( + token: string, + id: string, + manifest: Array<{ filename: string; path: string; checksum: string; size: number }> +) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/${id}/sync/diff`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + }, + body: JSON.stringify({ manifest }) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .then((json) => { + return json; + }) + .catch((err) => { + error = err.detail; + console.error(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const syncKnowledgeCleanup = async ( + token: string, + id: string, + fileIds: string[], + dirIds: string[] = [] +) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/${id}/sync/cleanup`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + }, + body: JSON.stringify({ file_ids: fileIds, dir_ids: dirIds }) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .then((json) => { + return json; + }) + .catch((err) => { + error = err.detail; + console.error(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + export const deleteKnowledgeById = async (token: string, id: string) => { let error = null; + const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/${id}/delete`, { method: 'DELETE', headers: { diff --git a/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte b/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte index 691e4f8c47..e92cb8fa6d 100644 --- a/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte +++ b/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte @@ -37,11 +37,14 @@ createKnowledgeDirectory, updateKnowledgeDirectory, deleteKnowledgeDirectory, - moveFileInKnowledge + moveFileInKnowledge, + syncKnowledgeDiff, + syncKnowledgeCleanup } from '$lib/apis/knowledge'; import { processWeb, processYoutubeVideo } from '$lib/apis/retrieval'; import { blobToFile, isYoutubeUrl, copyToClipboard } from '$lib/utils'; + import { computeFileHash } from '$lib/utils/hash'; import Spinner from '$lib/components/common/Spinner.svelte'; import Tooltip from '$lib/components/common/Tooltip.svelte'; @@ -75,6 +78,7 @@ let showNewDirectoryModal = false; let showSyncConfirmModal = false; + let pendingSyncFiles: Array<{path: string, filename: string, file: File}> | null = null; let showAccessControlModal = false; let minSize = 0; @@ -537,25 +541,171 @@ } }; - // Helper function to maintain file paths within zip - const syncDirectoryHandler = async () => { - if (fileItems.length > 0) { - const res = await resetKnowledgeById(localStorage.token, id).catch((e) => { - toast.error(`${e}`); - }); + // Collect files from a directory without uploading — returns {path, filename, file}[] + const collectDirectoryFiles = async (): Promise | null> => { + const isFileSystemAccessSupported = 'showDirectoryPicker' in window; - if (res) { - fileItems = []; - toast.success($i18n.t('Knowledge reset successfully.')); + try { + if (isFileSystemAccessSupported) { + const dirHandle = await window.showDirectoryPicker(); + const collected: Array<{path: string, filename: string, file: File}> = []; - // Upload directory - uploadDirectoryHandler(); + async function traverse(handle: FileSystemDirectoryHandle, dirPath = '') { + for await (const entry of handle.values()) { + if (entry.name.startsWith('.')) continue; + const entryPath = dirPath ? `${dirPath}/${entry.name}` : entry.name; + if (hasHiddenFolder(entryPath)) continue; + + if (entry.kind === 'file') { + const file = await entry.getFile(); + collected.push({ path: dirPath, filename: entry.name, file }); + } else if (entry.kind === 'directory') { + await traverse(entry, entryPath); + } + } + } + + await traverse(dirHandle); + return collected; + } else { + // Firefox fallback + return new Promise((resolve, reject) => { + const input = document.createElement('input'); + input.type = 'file'; + input.webkitdirectory = true; + input.directory = true; + input.multiple = true; + input.style.display = 'none'; + document.body.appendChild(input); + + input.onchange = () => { + try { + const files = Array.from(input.files || []) + .filter((file) => !hasHiddenFolder(file.webkitRelativePath) && !file.name.startsWith('.')); + + const collected = files.map((file) => { + const parts = file.webkitRelativePath.split('/'); + // Remove root dir name, extract path and filename + const withoutRoot = parts.slice(1); + const filename = withoutRoot.pop() || file.name; + const path = withoutRoot.join('/'); + return { path, filename, file }; + }); + + document.body.removeChild(input); + resolve(collected); + } catch (error) { + document.body.removeChild(input); + reject(error); + } + }; + + input.onerror = (error) => { + document.body.removeChild(input); + reject(error); + }; + + input.click(); + }); } - } else { - uploadDirectoryHandler(); + } catch (error) { + handleUploadError(error); + return null; } }; + // Incremental sync: hash locally → diff on server → upload only what changed + const syncDirectoryHandler = async () => { + if (!pendingSyncFiles?.length) return; + + // ── 2. Compute checksums ── + toast.info($i18n.t('Computing checksums...')); + const manifest = await Promise.all( + pendingSyncFiles.map(async (entry) => ({ + ...entry, + checksum: await computeFileHash(entry.file), + size: entry.file.size + })) + ); + pendingSyncFiles = null; + + // ── 3. Diff against knowledge base ── + toast.info($i18n.t('Comparing with knowledge base...')); + const diff = await syncKnowledgeDiff( + localStorage.token, + id, + manifest.map(({ filename, path, checksum, size }) => ({ filename, path, checksum, size })) + ); + + if (!diff) { + toast.error($i18n.t('Failed to compare files.')); + return; + } + + // ── 4. mkdir — create missing directories (parents first) ── + const createdDirectoryIds: Record = {}; + for (const dirPath of diff.mkdir) { + const segments = dirPath.split('/'); + const name = segments.at(-1)!; + const parentPath = segments.slice(0, -1).join('/'); + const parentId = parentPath ? createdDirectoryIds[parentPath] : null; + + const directory = await createKnowledgeDirectory( + localStorage.token, knowledge.id, name, parentId + ); + if (directory) { + createdDirectoryIds[dirPath] = directory.id; + } + } + + // ── 5. Upload added + modified files ── + const filesToUpload = manifest.filter((entry) => + diff.added.some((a: any) => a.filename === entry.filename && a.path === entry.path) || + diff.modified.some((m: any) => m.filename === entry.filename && m.path === entry.path) + ); + + let uploadedCount = 0; + for (const entry of filesToUpload) { + const fileObject = new File([entry.file], entry.filename, { type: entry.file.type }); + await uploadFile(localStorage.token, fileObject, { + knowledge_id: knowledge.id, + file_hash: entry.checksum, + directory_id: entry.path ? createdDirectoryIds[entry.path] : null + }); + uploadedCount++; + toast.info( + $i18n.t('Uploading: {{current}}/{{total}}', { + current: uploadedCount, + total: filesToUpload.length + }) + ); + } + + // ── 6. Cleanup — remove deleted files + rmdir orphaned directories ── + const staleFileIds = [ + ...diff.deleted.map((d: any) => d.file_id), + ...diff.modified.map((m: any) => m.stale_file_id) + ]; + + if (staleFileIds.length > 0 || diff.rmdir.length > 0) { + await syncKnowledgeCleanup(localStorage.token, id, staleFileIds, diff.rmdir); + } + + // ── 7. Report ── + toast.success( + $i18n.t( + 'Sync complete: {{added}} added, {{modified}} modified, {{deleted}} deleted, {{unmodified}} unmodified', + { + added: diff.added.length, + modified: diff.modified.length, + deleted: diff.deleted.length, + unmodified: diff.unmodified_count + } + ) + ); + init(); + }; + const addFileHandler = async (fileId) => { const res = await addFileToKnowledgeById( localStorage.token, @@ -924,11 +1074,15 @@ { syncDirectoryHandler(); }} + on:cancel={() => { + pendingSyncFiles = null; + }} /> { - showSyncConfirmModal = true; + onSync={async () => { + pendingSyncFiles = await collectDirectoryFiles(); + if (pendingSyncFiles?.length) { + showSyncConfirmModal = true; + } }} /> diff --git a/src/lib/components/workspace/Knowledge/KnowledgeBase/AddContentMenu.svelte b/src/lib/components/workspace/Knowledge/KnowledgeBase/AddContentMenu.svelte index 62b9f4f4e4..6a8cd25a85 100644 --- a/src/lib/components/workspace/Knowledge/KnowledgeBase/AddContentMenu.svelte +++ b/src/lib/components/workspace/Knowledge/KnowledgeBase/AddContentMenu.svelte @@ -90,7 +90,7 @@ diff --git a/src/lib/utils/hash.ts b/src/lib/utils/hash.ts new file mode 100644 index 0000000000..35af78041d --- /dev/null +++ b/src/lib/utils/hash.ts @@ -0,0 +1,115 @@ +/** + * Compute SHA-256 checksum of a File. + * + * Uses the Web Crypto API when available (HTTPS / localhost). + * Falls back to a pure-JS implementation for plain HTTP deployments + * where crypto.subtle is unavailable. + * + * @returns lowercase hex-encoded SHA-256 digest + */ +export async function computeFileHash(file: File): Promise { + const buffer = await file.arrayBuffer(); + + if (globalThis.crypto?.subtle) { + const digest = await crypto.subtle.digest('SHA-256', buffer); + return hexEncode(new Uint8Array(digest)); + } + + return sha256Fallback(new Uint8Array(buffer)); +} + +function hexEncode(bytes: Uint8Array): string { + const hex: string[] = new Array(bytes.length); + for (let i = 0; i < bytes.length; i++) { + hex[i] = bytes[i].toString(16).padStart(2, '0'); + } + return hex.join(''); +} + +// ── Pure-JS SHA-256 (RFC 6234) ────────────────────────────────────────────── +// Used only when crypto.subtle is unavailable (HTTP deployments). + +const K: Uint32Array = new Uint32Array([ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, + 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, + 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, + 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, + 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, + 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, + 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, + 0xc67178f2 +]); + +function rotr(n: number, x: number): number { + return (x >>> n) | (x << (32 - n)); +} + +function sha256Fallback(data: Uint8Array): string { + // Pre-processing: padding + const bitLen = data.length * 8; + const padLen = (((data.length + 8) >>> 6) + 1) << 6; + const padded = new Uint8Array(padLen); + padded.set(data); + padded[data.length] = 0x80; + + // Append original length in bits as 64-bit big-endian + const view = new DataView(padded.buffer); + view.setUint32(padLen - 4, bitLen, false); + + // Initial hash values + let h0 = 0x6a09e667; + let h1 = 0xbb67ae85; + let h2 = 0x3c6ef372; + let h3 = 0xa54ff53a; + let h4 = 0x510e527f; + let h5 = 0x9b05688c; + let h6 = 0x1f83d9ab; + let h7 = 0x5be0cd19; + + const w = new Uint32Array(64); + + for (let offset = 0; offset < padLen; offset += 64) { + // Prepare message schedule + for (let i = 0; i < 16; i++) { + w[i] = view.getUint32(offset + i * 4, false); + } + for (let i = 16; i < 64; i++) { + const s0 = rotr(7, w[i - 15]) ^ rotr(18, w[i - 15]) ^ (w[i - 15] >>> 3); + const s1 = rotr(17, w[i - 2]) ^ rotr(19, w[i - 2]) ^ (w[i - 2] >>> 10); + w[i] = (w[i - 16] + s0 + w[i - 7] + s1) | 0; + } + + let a = h0, b = h1, c = h2, d = h3; + let e = h4, f = h5, g = h6, h = h7; + + for (let i = 0; i < 64; i++) { + const S1 = rotr(6, e) ^ rotr(11, e) ^ rotr(25, e); + const ch = (e & f) ^ (~e & g); + const temp1 = (h + S1 + ch + K[i] + w[i]) | 0; + const S0 = rotr(2, a) ^ rotr(13, a) ^ rotr(22, a); + const maj = (a & b) ^ (a & c) ^ (b & c); + const temp2 = (S0 + maj) | 0; + + h = g; g = f; f = e; + e = (d + temp1) | 0; + d = c; c = b; b = a; + a = (temp1 + temp2) | 0; + } + + h0 = (h0 + a) | 0; h1 = (h1 + b) | 0; + h2 = (h2 + c) | 0; h3 = (h3 + d) | 0; + h4 = (h4 + e) | 0; h5 = (h5 + f) | 0; + h6 = (h6 + g) | 0; h7 = (h7 + h) | 0; + } + + const result = new Uint8Array(32); + const out = new DataView(result.buffer); + out.setUint32(0, h0, false); out.setUint32(4, h1, false); + out.setUint32(8, h2, false); out.setUint32(12, h3, false); + out.setUint32(16, h4, false); out.setUint32(20, h5, false); + out.setUint32(24, h6, false); out.setUint32(28, h7, false); + + return hexEncode(result); +}