From 0800c21c64c64810f24c2ec88cca5e36daebb10e Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Wed, 5 Aug 2026 00:47:49 -0500 Subject: [PATCH] refac --- backend/open_webui/models/chats.py | 162 +++++++++++++++++----------- backend/open_webui/routers/chats.py | 50 +++++---- backend/open_webui/tools/builtin.py | 45 +++++--- 3 files changed, 159 insertions(+), 98 deletions(-) diff --git a/backend/open_webui/models/chats.py b/backend/open_webui/models/chats.py index 25890a1b5c..3c6f8be16b 100644 --- a/backend/open_webui/models/chats.py +++ b/backend/open_webui/models/chats.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +import re import time import uuid @@ -40,6 +41,62 @@ from sqlalchemy.sql.expression import bindparam log = logging.getLogger(__name__) ACTIVE_CHAT_GAP_SECONDS = 30 * 60 +CHAT_SEARCH_FILTER_PREFIXES = ('tag:', 'folder:', 'pinned:', 'archived:', 'shared:') + + +def chat_search_content_query(text: str) -> str: + words = sanitize_text_for_db(text).lower().strip().split() + return ' '.join(word for word in words if not word.startswith(CHAT_SEARCH_FILTER_PREFIXES)).strip() + + +def chat_search_terms(text: str) -> list[str]: + return list(dict.fromkeys(re.findall(r'[a-z0-9]+', text.lower()))) + + +def chat_search_message_content_match_sql(dialect_name: str, key: str) -> str: + if dialect_name == 'sqlite': + return f""" + ( + EXISTS ( + SELECT 1 + FROM json_each(Chat.chat, '$.history.messages') AS history_message + WHERE LOWER(history_message.value->>'content') LIKE '%' || :{key} || '%' + ) + OR EXISTS ( + SELECT 1 + FROM json_each(Chat.chat, '$.messages') AS legacy_message + WHERE LOWER(legacy_message.value->>'content') LIKE '%' || :{key} || '%' + ) + ) + """ + + if dialect_name == 'postgresql': + return f""" + ( + EXISTS ( + SELECT 1 + FROM chat_message AS message + WHERE message.chat_id = Chat.id + AND message.user_id = Chat.user_id + AND json_typeof(message.content) = 'string' + AND LOWER(message.content #>> '{{}}') LIKE '%' || :{key} || '%' + ) + OR EXISTS ( + SELECT 1 + FROM json_each(Chat.chat#>'{{history,messages}}') AS history_message + WHERE json_typeof(history_message.value->'content') = 'string' + AND LOWER(history_message.value->>'content') LIKE '%' || :{key} || '%' + ) + OR EXISTS ( + SELECT 1 + FROM json_array_elements(Chat.chat->'messages') AS legacy_message + WHERE json_typeof(legacy_message->'content') = 'string' + AND LOWER(legacy_message->>'content') LIKE '%' || :{key} || '%' + ) + ) + """ + + raise NotImplementedError(f'Unsupported dialect: {dialect_name}') def chat_list_order(sort_by: str = 'updated_at', sort_dir: str = 'desc', user_id: str | None = None): @@ -1731,7 +1788,7 @@ class ChatTable: return [ChatModel.model_validate(chat) for chat in result.scalars().all()] # search user conversations - async def get_chats_by_user_id_and_search_text( + async def get_chats_by_user_id_and_search_text( # noqa: C901 self, user_id: str, search_text: str, @@ -1750,7 +1807,7 @@ class ChatTable: user_id, include_archived, filter={}, skip=skip, limit=limit, db=db ) - search_text_words = search_text.split(' ') + search_text_words = search_text.split() # search_text might contain 'tag:tag_name' format so we need to extract the tag_name tag_ids = [ @@ -1782,19 +1839,10 @@ class ChatTable: elif 'shared:false' in search_text_words: is_shared = False - search_text_words = [ - word - for word in search_text_words - if ( - not word.startswith('tag:') - and not word.startswith('folder:') - and not word.startswith('pinned:') - and not word.startswith('archived:') - and not word.startswith('shared:') - ) - ] + search_text_words = [word for word in search_text_words if not word.startswith(CHAT_SEARCH_FILTER_PREFIXES)] - search_text = ' '.join(search_text_words) + phrase_query = ' '.join(search_text_words).strip() + search_terms = chat_search_terms(phrase_query) async with get_async_db_context(db) as session: stmt = select(Chat).filter(Chat.user_id == user_id) @@ -1817,27 +1865,43 @@ class ChatTable: if folder_ids: stmt = stmt.filter(Chat.folder_id.in_(folder_ids)) - stmt = stmt.order_by(Chat.updated_at.desc(), Chat.id) - # Check if the database dialect is either 'sqlite' or 'postgresql' bind = await session.connection() dialect_name = bind.dialect.name - if dialect_name == 'sqlite': - # SQLite case: using JSON1 extension for JSON searching - sqlite_content_sql = ( - 'EXISTS (' - ' SELECT 1 ' - " FROM json_each(Chat.chat, '$.messages') AS message " - " WHERE LOWER(message.value->>'content') LIKE '%' || :content_key || '%'" - ')' + + search_params = {} + exact_match_clause = None + if phrase_query: + exact_match_clause = or_( + Chat.title.ilike(bindparam('phrase_title_key')), + text(chat_search_message_content_match_sql(dialect_name, 'phrase_content_key')), ) - sqlite_content_clause = text(sqlite_content_sql) - stmt = stmt.filter( - or_(Chat.title.ilike(bindparam('title_key')), sqlite_content_clause).params( - title_key=f'%{search_text}%', content_key=search_text - ) + search_params.update( + { + 'phrase_title_key': f'%{phrase_query}%', + 'phrase_content_key': phrase_query, + } ) + term_clauses = [] + for term_idx, term in enumerate(search_terms): + title_key = f'term_title_key_{term_idx}' + content_key = f'term_content_key_{term_idx}' + term_clauses.append( + or_( + Chat.title.ilike(bindparam(title_key)), + text(chat_search_message_content_match_sql(dialect_name, content_key)), + ) + ) + search_params[title_key] = f'%{term}%' + search_params[content_key] = term + + if term_clauses: + stmt = stmt.filter(or_(exact_match_clause, and_(*term_clauses))) + else: + stmt = stmt.filter(exact_match_clause) + + if dialect_name == 'sqlite': # Check if there are any tags to filter if 'none' in tag_ids: stmt = stmt.filter( @@ -1871,38 +1935,6 @@ class ChatTable: # Safety filter: title must not contain actual null bytes stmt = stmt.filter(text("Chat.title::text NOT LIKE '%\\x00%'")) - postgres_content_sql = """ - EXISTS ( - SELECT 1 - FROM chat_message AS message - WHERE message.chat_id = Chat.id - AND message.user_id = Chat.user_id - AND json_typeof(message.content) = 'string' - AND LOWER(message.content #>> '{}') LIKE '%' || :content_key || '%' - ) - OR EXISTS ( - SELECT 1 - FROM json_each(Chat.chat#>'{history,messages}') AS history_message - WHERE json_typeof(history_message.value->'content') = 'string' - AND LOWER(history_message.value->>'content') LIKE '%' || :content_key || '%' - ) - OR EXISTS ( - SELECT 1 - FROM json_array_elements(Chat.chat->'messages') AS legacy_message - WHERE json_typeof(legacy_message->'content') = 'string' - AND LOWER(legacy_message->>'content') LIKE '%' || :content_key || '%' - ) - """ - - postgres_content_clause = text(postgres_content_sql) - - stmt = stmt.filter( - or_( - Chat.title.ilike(bindparam('title_key')), - postgres_content_clause, - ) - ).params(title_key=f'%{search_text}%', content_key=search_text.lower()) - if 'none' in tag_ids: stmt = stmt.filter( text(""" @@ -1930,6 +1962,14 @@ class ChatTable: else: raise NotImplementedError(f'Unsupported dialect: {dialect_name}') + if exact_match_clause is not None: + stmt = stmt.order_by(case((exact_match_clause, 0), else_=1), Chat.updated_at.desc(), Chat.id) + else: + stmt = stmt.order_by(Chat.updated_at.desc(), Chat.id) + + if search_params: + stmt = stmt.params(**search_params) + # Perform pagination at the SQL level stmt = stmt.offset(skip).limit(limit) result = await session.execute(stmt) diff --git a/backend/open_webui/routers/chats.py b/backend/open_webui/routers/chats.py index 0c46f5bbbf..bd2801bbe3 100644 --- a/backend/open_webui/routers/chats.py +++ b/backend/open_webui/routers/chats.py @@ -13,7 +13,6 @@ from open_webui.constants import ERROR_MESSAGES from open_webui.events import EVENTS, publish_event from open_webui.internal.db import get_async_session from open_webui.models.access_grants import AccessGrants -from open_webui.models.config import Config from open_webui.models.chat_messages import ChatMessages from open_webui.models.chats import ( AggregateChatStats, @@ -27,9 +26,12 @@ from open_webui.models.chats import ( ChatStatsExport, ChatTitleIdResponse, ChatUsageStatsListResponse, - is_internal_chat, MessageStats, + chat_search_content_query, + chat_search_terms, + is_internal_chat, ) +from open_webui.models.config import Config from open_webui.models.folders import Folders from open_webui.models.shared_chats import SharedChatResponse, SharedChats from open_webui.models.tags import TagModel, Tags @@ -49,8 +51,6 @@ log = logging.getLogger(__name__) router = APIRouter() -SEARCH_FILTER_PREFIXES = ('tag:', 'folder:', 'pinned:', 'archived:', 'shared:') - CHAT_CONFIG_KEYS = { 'CONTEXT_COMPACTION_MODEL': 'chat.context_compaction.model', 'ENABLE_CONTEXT_COMPACTION': 'chat.context_compaction.enable', @@ -147,38 +147,42 @@ class CompactChatForm(BaseModel): def chat_search_content_text(text: str) -> str: - words = text.lower().strip().split(' ') - return ' '.join(word for word in words if not word.startswith(SEARCH_FILTER_PREFIXES)).strip() + return chat_search_content_query(text) def chat_search_snippet(chat: dict, search_text: str, max_length: int = 200) -> str | None: if not search_text: return None - messages = chat.get('messages', []) + history = chat.get('history', {}) + messages = history.get('messages') if isinstance(history, dict) else None + if not messages: + messages = chat.get('messages', []) or [] if isinstance(messages, dict): messages = messages.values() - for message in messages: - if not isinstance(message, dict): - continue + needles = list(dict.fromkeys([search_text, *chat_search_terms(search_text)])) + for needle in needles: + for message in messages: + if not isinstance(message, dict): + continue - content = message.get('content') - if not isinstance(content, str): - continue + content = message.get('content') + if not isinstance(content, str): + continue - index = content.lower().find(search_text) - if index == -1: - continue + index = content.lower().find(needle) + if index == -1: + continue - start = max(index - max_length // 2, 0) - end = min(start + max_length, len(content)) - if index + len(search_text) > end: - end = min(index + len(search_text), len(content)) - start = max(end - max_length, 0) + start = max(index - max_length // 2, 0) + end = min(start + max_length, len(content)) + if index + len(needle) > end: + end = min(index + len(needle), len(content)) + start = max(end - max_length, 0) - snippet = ' '.join(content[start:end].split()) - return f'{"..." if start else ""}{snippet}{"..." if end < len(content) else ""}' + snippet = ' '.join(content[start:end].split()) + return f'{"..." if start else ""}{snippet}{"..." if end < len(content) else ""}' return None diff --git a/backend/open_webui/tools/builtin.py b/backend/open_webui/tools/builtin.py index 94fae4af43..c71b305898 100644 --- a/backend/open_webui/tools/builtin.py +++ b/backend/open_webui/tools/builtin.py @@ -21,7 +21,7 @@ from open_webui.env import ( ) from open_webui.events import EVENTS, publish_event from open_webui.models.channels import Channel, ChannelMember, Channels -from open_webui.models.chats import Chats +from open_webui.models.chats import Chats, chat_search_content_query, chat_search_terms from open_webui.models.config import Config from open_webui.models.groups import Groups from open_webui.models.memories import Memories @@ -1384,10 +1384,13 @@ async def search_chats( __chat_id__: str = None, ) -> str: """ - Search the user's previous chat conversations by title and message content. - Helpful for finding details from earlier conversations. + Search the user's previous chat conversations by title and message content, + excluding the current chat. Helpful for finding details from earlier + conversations when they are not already visible in the current context. + Exact phrase matches are preferred, and descriptive keyword queries are + supported. - :param query: The search query to find matching chats + :param query: Exact phrase or descriptive keyword query to find matching previous chats :param count: Maximum number of results to return (default: 5) :param start_timestamp: Only include chats updated after this Unix timestamp (seconds) :param end_timestamp: Only include chats updated before this Unix timestamp (seconds) @@ -1425,19 +1428,33 @@ async def search_chats( # Find a matching message snippet snippet = '' messages = (getattr(chat, 'chat', None) or {}).get('history', {}).get('messages', {}) - lower_query = query.lower() + if not messages: + messages = (getattr(chat, 'chat', None) or {}).get('messages', {}) or {} + if isinstance(messages, list): + messages = {str(idx): message for idx, message in enumerate(messages)} - for msg_id, msg in messages.items(): - content = msg.get('content', '') - if isinstance(content, str) and lower_query in content.lower(): - idx = content.lower().find(lower_query) - start = max(0, idx - 50) - end = min(len(content), idx + len(query) + 100) - snippet = ('...' if start > 0 else '') + content[start:end] + ('...' if end < len(content) else '') + lower_query = chat_search_content_query(query) + needles = list(dict.fromkeys([lower_query, *chat_search_terms(lower_query)])) if lower_query else [] + + for needle in needles: + for msg_id, msg in messages.items(): + content = msg.get('content', '') if isinstance(msg, dict) else '' + if isinstance(content, str) and needle in content.lower(): + idx = content.lower().find(needle) + start = max(0, idx - 50) + end = min(len(content), idx + len(needle) + 100) + snippet = ( + ('...' if start > 0 else '') + + content[start:end] + + ('...' if end < len(content) else '') + ) + break + if snippet: break - if not snippet and lower_query in chat.title.lower(): - snippet = f'Title match: {chat.title}' + title = chat.title or '' + if not snippet and any(needle in title.lower() for needle in needles): + snippet = f'Title match: {title}' results.append( {