Mirrors/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2025-12-15 19:37:47 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix: tiktoken encoding model issue

Browse Source
This commit is contained in:
Timothy J. Baek
2024-10-25 21:38:28 -07:00
parent adede5480d
commit 50dcad0f73
3 changed files with 11 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
backend/open_webui/apps/retrieval/main.py
View File

@@ -50,7 +50,7 @@ from open_webui.apps.retrieval.utils import (
from open_webui.apps.webui.models.files import Files
from open_webui.config import (
BRAVE_SEARCH_API_KEY,
TIKTOKEN_ENCODING_NAME,
TIKTOKEN_ENCODING_MODEL_NAME,
RAG_TEXT_SPLITTER,
CHUNK_OVERLAP,
CHUNK_SIZE,
@@ -135,7 +135,7 @@ app.state.config.CONTENT_EXTRACTION_ENGINE = CONTENT_EXTRACTION_ENGINE
app.state.config.TIKA_SERVER_URL = TIKA_SERVER_URL
app.state.config.TEXT_SPLITTER = RAG_TEXT_SPLITTER
app.state.config.TIKTOKEN_ENCODING_NAME = TIKTOKEN_ENCODING_NAME
app.state.config.TIKTOKEN_ENCODING_MODEL_NAME = TIKTOKEN_ENCODING_MODEL_NAME
app.state.config.CHUNK_SIZE = CHUNK_SIZE
app.state.config.CHUNK_OVERLAP = CHUNK_OVERLAP
@@ -667,7 +667,7 @@ def save_docs_to_vector_db(
)
elif app.state.config.TEXT_SPLITTER == "token":
text_splitter = TokenTextSplitter(
encoding_name=app.state.config.TIKTOKEN_ENCODING_NAME,
model_name=app.state.config.TIKTOKEN_ENCODING_MODEL_NAME,
chunk_size=app.state.config.CHUNK_SIZE,
chunk_overlap=app.state.config.CHUNK_OVERLAP,
add_start_index=True,

8
backend/open_webui/config.py
View File

@@ -1074,10 +1074,10 @@ RAG_TEXT_SPLITTER = PersistentConfig(
TIKTOKEN_CACHE_DIR = os.environ.get("TIKTOKEN_CACHE_DIR", f"{CACHE_DIR}/tiktoken")
TIKTOKEN_ENCODING_NAME = PersistentConfig(
"TIKTOKEN_ENCODING_NAME",
"rag.tiktoken_encoding_name",
os.environ.get("TIKTOKEN_ENCODING_NAME", "cl100k_base"),
TIKTOKEN_ENCODING_MODEL_NAME = PersistentConfig(
"TIKTOKEN_ENCODING_MODEL_NAME",
"rag.tiktoken_encoding_model_name",
os.environ.get("TIKTOKEN_ENCODING_MODEL_NAME", "cl100k_base"),
)