enh: vector db hash collision check

backend/open_webui/apps/retrieval/main.py

@@ -641,6 +641,16 @@ def save_docs_to_vector_db(
 ) -> bool:
     log.info(f"save_docs_to_vector_db {docs} {collection_name}")
 
+    # Check if entries with the same hash (metadata.hash) already exist
+    if metadata and "hash" in metadata:
+        existing_docs = VECTOR_DB_CLIENT.query(
+            collection_name=collection_name,
+            filter={"hash": metadata["hash"]},
+        )
+        if existing_docs:
+            log.info(f"Document with hash {metadata['hash']} already exists")
+            return True
+
     if split:
         text_splitter = RecursiveCharacterTextSplitter(
             chunk_size=app.state.config.CHUNK_SIZE,