Merge pull request #5861 from open-webui/projects

feat: knowledge/projects

backend/open_webui/apps/webui/main.py

@@ -10,11 +10,11 @@ from open_webui.apps.webui.routers import (
     auths,
     chats,
     configs,
-    documents,
     files,
     functions,
     memories,
     models,
+    knowledge,
     prompts,
     tools,
     users,
@@ -111,15 +111,15 @@ app.include_router(auths.router, prefix="/auths", tags=["auths"])
 app.include_router(users.router, prefix="/users", tags=["users"])
 app.include_router(chats.router, prefix="/chats", tags=["chats"])
 
-app.include_router(documents.router, prefix="/documents", tags=["documents"])
 app.include_router(models.router, prefix="/models", tags=["models"])
+app.include_router(knowledge.router, prefix="/knowledge", tags=["knowledge"])
 app.include_router(prompts.router, prefix="/prompts", tags=["prompts"])
 
-app.include_router(memories.router, prefix="/memories", tags=["memories"])
 app.include_router(files.router, prefix="/files", tags=["files"])
 app.include_router(tools.router, prefix="/tools", tags=["tools"])
 app.include_router(functions.router, prefix="/functions", tags=["functions"])
 
+app.include_router(memories.router, prefix="/memories", tags=["memories"])
 app.include_router(utils.router, prefix="/utils", tags=["utils"])
 
 

backend/open_webui/apps/webui/models/files.py

@@ -5,7 +5,7 @@ from typing import Optional
 from open_webui.apps.webui.internal.db import Base, JSONField, get_db
 from open_webui.env import SRC_LOG_LEVELS
 from pydantic import BaseModel, ConfigDict
-from sqlalchemy import BigInteger, Column, String, Text
+from sqlalchemy import BigInteger, Column, String, Text, JSON
 
 log = logging.getLogger(__name__)
 log.setLevel(SRC_LOG_LEVELS["MODELS"])
@@ -20,19 +20,29 @@ class File(Base):
 
     id = Column(String, primary_key=True)
     user_id = Column(String)
+    hash = Column(Text, nullable=True)
+
     filename = Column(Text)
+    data = Column(JSON, nullable=True)
     meta = Column(JSONField)
+
     created_at = Column(BigInteger)
+    updated_at = Column(BigInteger)
 
 
 class FileModel(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+
     id: str
     user_id: str
-    filename: str
-    meta: dict
-    created_at: int  # timestamp in epoch
+    hash: Optional[str] = None
 
-    model_config = ConfigDict(from_attributes=True)
+    filename: str
+    data: Optional[dict] = None
+    meta: dict
+
+    created_at: int  # timestamp in epoch
+    updated_at: int  # timestamp in epoch
 
 
 ####################
@@ -43,14 +53,21 @@ class FileModel(BaseModel):
 class FileModelResponse(BaseModel):
     id: str
     user_id: str
+    hash: Optional[str] = None
+
     filename: str
+    data: Optional[dict] = None
     meta: dict
+
     created_at: int  # timestamp in epoch
+    updated_at: int  # timestamp in epoch
 
 
 class FileForm(BaseModel):
     id: str
+    hash: Optional[str] = None
     filename: str
+    data: dict = {}
     meta: dict = {}
 
 
@@ -62,6 +79,7 @@ class FilesTable:
                     **form_data.model_dump(),
                     "user_id": user_id,
                     "created_at": int(time.time()),
+                    "updated_at": int(time.time()),
                 }
             )
 
@@ -90,6 +108,13 @@ class FilesTable:
         with get_db() as db:
             return [FileModel.model_validate(file) for file in db.query(File).all()]
 
+    def get_files_by_ids(self, ids: list[str]) -> list[FileModel]:
+        with get_db() as db:
+            return [
+                FileModel.model_validate(file)
+                for file in db.query(File).filter(File.id.in_(ids)).all()
+            ]
+
     def get_files_by_user_id(self, user_id: str) -> list[FileModel]:
         with get_db() as db:
             return [
@@ -97,17 +122,38 @@ class FilesTable:
                 for file in db.query(File).filter_by(user_id=user_id).all()
             ]
 
-    def update_files_metadata_by_id(self, id: str, meta: dict) -> Optional[FileModel]:
+    def update_file_hash_by_id(self, id: str, hash: str) -> Optional[FileModel]:
         with get_db() as db:
             try:
                 file = db.query(File).filter_by(id=id).first()
-                file.meta = {**file.meta, **meta}
+                file.hash = hash
                 db.commit()
 
                 return FileModel.model_validate(file)
             except Exception:
                 return None
 
+    def update_file_data_by_id(self, id: str, data: dict) -> Optional[FileModel]:
+        with get_db() as db:
+            try:
+                file = db.query(File).filter_by(id=id).first()
+                file.data = {**(file.data if file.data else {}), **data}
+                db.commit()
+                return FileModel.model_validate(file)
+            except Exception as e:
+
+                return None
+
+    def update_file_metadata_by_id(self, id: str, meta: dict) -> Optional[FileModel]:
+        with get_db() as db:
+            try:
+                file = db.query(File).filter_by(id=id).first()
+                file.meta = {**(file.meta if file.meta else {}), **meta}
+                db.commit()
+                return FileModel.model_validate(file)
+            except Exception:
+                return None
+
     def delete_file_by_id(self, id: str) -> bool:
         with get_db() as db:
             try:

backend/open_webui/apps/webui/models/knowledge.py

@@ -0,0 +1,152 @@
+import json
+import logging
+import time
+from typing import Optional
+import uuid
+
+from open_webui.apps.webui.internal.db import Base, get_db
+from open_webui.env import SRC_LOG_LEVELS
+from pydantic import BaseModel, ConfigDict
+from sqlalchemy import BigInteger, Column, String, Text, JSON
+
+
+log = logging.getLogger(__name__)
+log.setLevel(SRC_LOG_LEVELS["MODELS"])
+
+####################
+# Knowledge DB Schema
+####################
+
+
+class Knowledge(Base):
+    __tablename__ = "knowledge"
+
+    id = Column(Text, unique=True, primary_key=True)
+    user_id = Column(Text)
+
+    name = Column(Text)
+    description = Column(Text)
+
+    data = Column(JSON, nullable=True)
+    meta = Column(JSON, nullable=True)
+
+    created_at = Column(BigInteger)
+    updated_at = Column(BigInteger)
+
+
+class KnowledgeModel(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+
+    id: str
+    user_id: str
+
+    name: str
+    description: str
+
+    data: Optional[dict] = None
+    meta: Optional[dict] = None
+
+    created_at: int  # timestamp in epoch
+    updated_at: int  # timestamp in epoch
+
+
+####################
+# Forms
+####################
+
+
+class KnowledgeResponse(BaseModel):
+    id: str
+    name: str
+    description: str
+    data: Optional[dict] = None
+    meta: Optional[dict] = None
+    created_at: int  # timestamp in epoch
+    updated_at: int  # timestamp in epoch
+
+
+class KnowledgeForm(BaseModel):
+    name: str
+    description: str
+    data: Optional[dict] = None
+
+
+class KnowledgeUpdateForm(BaseModel):
+    name: Optional[str] = None
+    description: Optional[str] = None
+    data: Optional[dict] = None
+
+
+class KnowledgeTable:
+    def insert_new_knowledge(
+        self, user_id: str, form_data: KnowledgeForm
+    ) -> Optional[KnowledgeModel]:
+        with get_db() as db:
+            knowledge = KnowledgeModel(
+                **{
+                    **form_data.model_dump(),
+                    "id": str(uuid.uuid4()),
+                    "user_id": user_id,
+                    "created_at": int(time.time()),
+                    "updated_at": int(time.time()),
+                }
+            )
+
+            try:
+                result = Knowledge(**knowledge.model_dump())
+                db.add(result)
+                db.commit()
+                db.refresh(result)
+                if result:
+                    return KnowledgeModel.model_validate(result)
+                else:
+                    return None
+            except Exception:
+                return None
+
+    def get_knowledge_items(self) -> list[KnowledgeModel]:
+        with get_db() as db:
+            return [
+                KnowledgeModel.model_validate(knowledge)
+                for knowledge in db.query(Knowledge)
+                .order_by(Knowledge.updated_at.desc())
+                .all()
+            ]
+
+    def get_knowledge_by_id(self, id: str) -> Optional[KnowledgeModel]:
+        try:
+            with get_db() as db:
+                knowledge = db.query(Knowledge).filter_by(id=id).first()
+                return KnowledgeModel.model_validate(knowledge) if knowledge else None
+        except Exception:
+            return None
+
+    def update_knowledge_by_id(
+        self, id: str, form_data: KnowledgeUpdateForm, overwrite: bool = False
+    ) -> Optional[KnowledgeModel]:
+        try:
+            with get_db() as db:
+                knowledge = self.get_knowledge_by_id(id=id)
+                db.query(Knowledge).filter_by(id=id).update(
+                    {
+                        **form_data.model_dump(exclude_none=True),
+                        "updated_at": int(time.time()),
+                    }
+                )
+                db.commit()
+                return self.get_knowledge_by_id(id=id)
+        except Exception as e:
+            log.exception(e)
+            return None
+
+    def delete_knowledge_by_id(self, id: str) -> bool:
+        try:
+            with get_db() as db:
+                db.query(Knowledge).filter_by(id=id).delete()
+                db.commit()
+                return True
+        except Exception:
+            return False
+
+
+Knowledges = KnowledgeTable()

backend/open_webui/apps/webui/routers/files.py

@@ -4,13 +4,18 @@ import shutil
 import uuid
 from pathlib import Path
 from typing import Optional
+from pydantic import BaseModel
 
 from open_webui.apps.webui.models.files import FileForm, FileModel, Files
+from open_webui.apps.retrieval.main import process_file, ProcessFileForm
+
 from open_webui.config import UPLOAD_DIR
 from open_webui.constants import ERROR_MESSAGES
 from open_webui.env import SRC_LOG_LEVELS
+
+
 from fastapi import APIRouter, Depends, File, HTTPException, UploadFile, status
-from fastapi.responses import FileResponse
+from fastapi.responses import FileResponse, StreamingResponse
 from open_webui.utils.utils import get_admin_user, get_verified_user
 
 log = logging.getLogger(__name__)
@@ -58,6 +63,13 @@ def upload_file(file: UploadFile = File(...), user=Depends(get_verified_user)):
             ),
         )
 
+        try:
+            process_file(ProcessFileForm(file_id=id))
+            file = Files.get_file_by_id(id=id)
+        except Exception as e:
+            log.exception(e)
+            log.error(f"Error processing file: {file.id}")
+
         if file:
             return file
         else:
@@ -143,6 +155,55 @@ async def get_file_by_id(id: str, user=Depends(get_verified_user)):
         )
 
 
+############################
+# Get File Data Content By Id
+############################
+
+
+@router.get("/{id}/data/content")
+async def get_file_data_content_by_id(id: str, user=Depends(get_verified_user)):
+    file = Files.get_file_by_id(id)
+
+    if file and (file.user_id == user.id or user.role == "admin"):
+        return {"content": file.data.get("content", "")}
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
+
+
+############################
+# Update File Data Content By Id
+############################
+
+
+class ContentForm(BaseModel):
+    content: str
+
+
+@router.post("/{id}/data/content/update")
+async def update_file_data_content_by_id(
+    id: str, form_data: ContentForm, user=Depends(get_verified_user)
+):
+    file = Files.get_file_by_id(id)
+
+    if file and (file.user_id == user.id or user.role == "admin"):
+        try:
+            process_file(ProcessFileForm(file_id=id, content=form_data.content))
+            file = Files.get_file_by_id(id=id)
+        except Exception as e:
+            log.exception(e)
+            log.error(f"Error processing file: {file.id}")
+
+        return {"content": file.data.get("content", "")}
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
+
+
 ############################
 # Get File Content By Id
 ############################
@@ -171,34 +232,37 @@ async def get_file_content_by_id(id: str, user=Depends(get_verified_user)):
         )
 
 
-@router.get("/{id}/content/text")
-async def get_file_text_content_by_id(id: str, user=Depends(get_verified_user)):
-    file = Files.get_file_by_id(id)
-
-    if file and (file.user_id == user.id or user.role == "admin"):
-        return {"text": file.meta.get("content", {}).get("text", None)}
-    else:
-        raise HTTPException(
-            status_code=status.HTTP_404_NOT_FOUND,
-            detail=ERROR_MESSAGES.NOT_FOUND,
-        )
-
-
 @router.get("/{id}/content/{file_name}", response_model=Optional[FileModel])
 async def get_file_content_by_id(id: str, user=Depends(get_verified_user)):
     file = Files.get_file_by_id(id)
 
     if file and (file.user_id == user.id or user.role == "admin"):
-        file_path = Path(file.meta["path"])
+        file_path = file.meta.get("path")
+        if file_path:
+            file_path = Path(file_path)
 
-        # Check if the file already exists in the cache
-        if file_path.is_file():
-            print(f"file_path: {file_path}")
-            return FileResponse(file_path)
+            # Check if the file already exists in the cache
+            if file_path.is_file():
+                print(f"file_path: {file_path}")
+                return FileResponse(file_path)
+            else:
+                raise HTTPException(
+                    status_code=status.HTTP_404_NOT_FOUND,
+                    detail=ERROR_MESSAGES.NOT_FOUND,
+                )
         else:
-            raise HTTPException(
-                status_code=status.HTTP_404_NOT_FOUND,
-                detail=ERROR_MESSAGES.NOT_FOUND,
+            # File path doesn’t exist, return the content as .txt if possible
+            file_content = file.content.get("content", "")
+            file_name = file.filename
+
+            # Create a generator that encodes the file content
+            def generator():
+                yield file_content.encode("utf-8")
+
+            return StreamingResponse(
+                generator(),
+                media_type="text/plain",
+                headers={"Content-Disposition": f"attachment; filename={file_name}"},
             )
     else:
         raise HTTPException(

backend/open_webui/apps/webui/routers/knowledge.py

@@ -0,0 +1,320 @@
+import json
+from typing import Optional, Union
+from pydantic import BaseModel
+from fastapi import APIRouter, Depends, HTTPException, status
+
+
+from open_webui.apps.webui.models.knowledge import (
+    Knowledges,
+    KnowledgeUpdateForm,
+    KnowledgeForm,
+    KnowledgeResponse,
+)
+from open_webui.apps.webui.models.files import Files, FileModel
+from open_webui.apps.retrieval.vector.connector import VECTOR_DB_CLIENT
+from open_webui.apps.retrieval.main import process_file, ProcessFileForm
+
+
+from open_webui.constants import ERROR_MESSAGES
+from open_webui.utils.utils import get_admin_user, get_verified_user
+
+
+router = APIRouter()
+
+############################
+# GetKnowledgeItems
+############################
+
+
+@router.get(
+    "/", response_model=Optional[Union[list[KnowledgeResponse], KnowledgeResponse]]
+)
+async def get_knowledge_items(
+    id: Optional[str] = None, user=Depends(get_verified_user)
+):
+    if id:
+        knowledge = Knowledges.get_knowledge_by_id(id=id)
+
+        if knowledge:
+            return knowledge
+        else:
+            raise HTTPException(
+                status_code=status.HTTP_401_UNAUTHORIZED,
+                detail=ERROR_MESSAGES.NOT_FOUND,
+            )
+    else:
+        return [
+            KnowledgeResponse(**knowledge.model_dump())
+            for knowledge in Knowledges.get_knowledge_items()
+        ]
+
+
+############################
+# CreateNewKnowledge
+############################
+
+
+@router.post("/create", response_model=Optional[KnowledgeResponse])
+async def create_new_knowledge(form_data: KnowledgeForm, user=Depends(get_admin_user)):
+    knowledge = Knowledges.insert_new_knowledge(user.id, form_data)
+
+    if knowledge:
+        return knowledge
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.FILE_EXISTS,
+        )
+
+
+############################
+# GetKnowledgeById
+############################
+
+
+class KnowledgeFilesResponse(KnowledgeResponse):
+    files: list[FileModel]
+
+
+@router.get("/{id}", response_model=Optional[KnowledgeFilesResponse])
+async def get_knowledge_by_id(id: str, user=Depends(get_verified_user)):
+    knowledge = Knowledges.get_knowledge_by_id(id=id)
+
+    if knowledge:
+        file_ids = knowledge.data.get("file_ids", []) if knowledge.data else []
+        files = Files.get_files_by_ids(file_ids)
+
+        return KnowledgeFilesResponse(
+            **knowledge.model_dump(),
+            files=files,
+        )
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
+
+
+############################
+# UpdateKnowledgeById
+############################
+
+
+@router.post("/{id}/update", response_model=Optional[KnowledgeFilesResponse])
+async def update_knowledge_by_id(
+    id: str,
+    form_data: KnowledgeUpdateForm,
+    user=Depends(get_admin_user),
+):
+    knowledge = Knowledges.update_knowledge_by_id(id=id, form_data=form_data)
+
+    if knowledge:
+        file_ids = knowledge.data.get("file_ids", []) if knowledge.data else []
+        files = Files.get_files_by_ids(file_ids)
+
+        return KnowledgeFilesResponse(
+            **knowledge.model_dump(),
+            files=files,
+        )
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.ID_TAKEN,
+        )
+
+
+############################
+# AddFileToKnowledge
+############################
+
+
+class KnowledgeFileIdForm(BaseModel):
+    file_id: str
+
+
+@router.post("/{id}/file/add", response_model=Optional[KnowledgeFilesResponse])
+def add_file_to_knowledge_by_id(
+    id: str,
+    form_data: KnowledgeFileIdForm,
+    user=Depends(get_admin_user),
+):
+    knowledge = Knowledges.get_knowledge_by_id(id=id)
+    file = Files.get_file_by_id(form_data.file_id)
+    if not file:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
+    if not file.data:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.FILE_NOT_PROCESSED,
+        )
+
+    # Add content to the vector database
+    try:
+        process_file(ProcessFileForm(file_id=form_data.file_id, collection_name=id))
+    except Exception as e:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=str(e),
+        )
+
+    if knowledge:
+        data = knowledge.data or {}
+        file_ids = data.get("file_ids", [])
+
+        if form_data.file_id not in file_ids:
+            file_ids.append(form_data.file_id)
+            data["file_ids"] = file_ids
+
+            knowledge = Knowledges.update_knowledge_by_id(
+                id=id, form_data=KnowledgeUpdateForm(data=data)
+            )
+
+            if knowledge:
+                files = Files.get_files_by_ids(file_ids)
+
+                return KnowledgeFilesResponse(
+                    **knowledge.model_dump(),
+                    files=files,
+                )
+            else:
+                raise HTTPException(
+                    status_code=status.HTTP_400_BAD_REQUEST,
+                    detail=ERROR_MESSAGES.DEFAULT("knowledge"),
+                )
+        else:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=ERROR_MESSAGES.DEFAULT("file_id"),
+            )
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
+
+
+@router.post("/{id}/file/update", response_model=Optional[KnowledgeFilesResponse])
+def update_file_from_knowledge_by_id(
+    id: str,
+    form_data: KnowledgeFileIdForm,
+    user=Depends(get_admin_user),
+):
+    knowledge = Knowledges.get_knowledge_by_id(id=id)
+    file = Files.get_file_by_id(form_data.file_id)
+    if not file:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
+
+    # Remove content from the vector database
+    VECTOR_DB_CLIENT.delete(
+        collection_name=knowledge.id, filter={"file_id": form_data.file_id}
+    )
+
+    # Add content to the vector database
+    try:
+        process_file(ProcessFileForm(file_id=form_data.file_id, collection_name=id))
+    except Exception as e:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=str(e),
+        )
+
+    if knowledge:
+        data = knowledge.data or {}
+        file_ids = data.get("file_ids", [])
+
+        files = Files.get_files_by_ids(file_ids)
+
+        return KnowledgeFilesResponse(
+            **knowledge.model_dump(),
+            files=files,
+        )
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
+
+
+############################
+# RemoveFileFromKnowledge
+############################
+
+
+@router.post("/{id}/file/remove", response_model=Optional[KnowledgeFilesResponse])
+def remove_file_from_knowledge_by_id(
+    id: str,
+    form_data: KnowledgeFileIdForm,
+    user=Depends(get_admin_user),
+):
+    knowledge = Knowledges.get_knowledge_by_id(id=id)
+    file = Files.get_file_by_id(form_data.file_id)
+    if not file:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
+
+    # Remove content from the vector database
+    VECTOR_DB_CLIENT.delete(
+        collection_name=knowledge.id, filter={"file_id": form_data.file_id}
+    )
+
+    result = VECTOR_DB_CLIENT.query(
+        collection_name=knowledge.id,
+        filter={"file_id": form_data.file_id},
+    )
+
+    Files.delete_file_by_id(form_data.file_id)
+
+    if knowledge:
+        data = knowledge.data or {}
+        file_ids = data.get("file_ids", [])
+
+        if form_data.file_id in file_ids:
+            file_ids.remove(form_data.file_id)
+            data["file_ids"] = file_ids
+
+            knowledge = Knowledges.update_knowledge_by_id(
+                id=id, form_data=KnowledgeUpdateForm(data=data)
+            )
+
+            if knowledge:
+                files = Files.get_files_by_ids(file_ids)
+
+                return KnowledgeFilesResponse(
+                    **knowledge.model_dump(),
+                    files=files,
+                )
+            else:
+                raise HTTPException(
+                    status_code=status.HTTP_400_BAD_REQUEST,
+                    detail=ERROR_MESSAGES.DEFAULT("knowledge"),
+                )
+        else:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=ERROR_MESSAGES.DEFAULT("file_id"),
+            )
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
+
+
+############################
+# DeleteKnowledgeById
+############################
+
+
+@router.delete("/{id}/delete", response_model=bool)
+async def delete_knowledge_by_id(id: str, user=Depends(get_admin_user)):
+    VECTOR_DB_CLIENT.delete_collection(collection_name=id)
+    result = Knowledges.delete_knowledge_by_id(id=id)
+    return result