Mirrors/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2025-12-15 11:27:46 +01:00
Code Issues Packages Projects Releases Wiki Activity

added new feature : oracle23ai vector search

Browse Source
This commit is contained in:
Oracle Public Cloud User
2025-07-07 12:13:05 +00:00
parent b56dbb26be
commit 25e241ae41
7 changed files with 9311 additions and 2546 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
backend/open_webui/config.py
View File

@@ -1855,12 +1855,27 @@ ORACLE_WALLET_DIR = os.environ.get("ORACLE_WALLET_DIR", None)
ORACLE_WALLET_PASSWORD = os.environ.get("ORACLE_WALLET_PASSWORD", None) ORACLE_WALLET_PASSWORD = os.environ.get("ORACLE_WALLET_PASSWORD", None)
ORACLE_VECTOR_LENGTH = os.environ.get("ORACLE_VECTOR_LENGTH", 768) ORACLE_VECTOR_LENGTH = os.environ.get("ORACLE_VECTOR_LENGTH", 768)
ORACLE_DB_POOL_MIN = int(os.environ.get("ORACLE_DB_POOL_MIN", 2))
ORACLE_DB_POOL_MAX = int(os.environ.get("ORACLE_DB_POOL_MAX", 10))
ORACLE_DB_POOL_INCREMENT = int(os.environ.get("ORACLE_DB_POOL_INCREMENT", 1))
log.info(f"VECTOR_DB: {VECTOR_DB}")
log.info(f"ORACLE_DB_USE_WALLET: {ORACLE_DB_USE_WALLET}/type: {type(ORACLE_DB_USE_WALLET)}")
log.info(f"ORACLE_DB_USER: {ORACLE_DB_USER}/type: {type(ORACLE_DB_USER)}")
log.info(f"ORACLE_DB_PASSWORD: {ORACLE_DB_PASSWORD}/type: {type(ORACLE_DB_PASSWORD)}")
log.info(f"ORACLE_DB_DSN: {ORACLE_DB_DSN}/type: {type(ORACLE_DB_DSN)}")
log.info(f"ORACLE_WALLET_DIR: {ORACLE_WALLET_DIR}/type: {type(ORACLE_WALLET_DIR)}")
log.info(f"ORACLE_WALLET_PASSWORD: {ORACLE_WALLET_PASSWORD}/type: {type(ORACLE_WALLET_PASSWORD)}")
log.info(f"ORACLE_VECTOR_LENGTH: {ORACLE_VECTOR_LENGTH}")
log.info(f"ORACLE_DB_POOL_MIN: {ORACLE_DB_POOL_MIN}")
log.info(f"ORACLE_DB_POOL_MAX: {ORACLE_DB_POOL_MAX}")
log.info(f"ORACLE_DB_POOL_INCREMENT: {ORACLE_DB_POOL_INCREMENT}")
if VECTOR_DB == "oracle23ai" and not ORACLE_DB_USER or not ORACLE_DB_PASSWORD or not ORACLE_DB_DSN: if VECTOR_DB == "oracle23ai" and not ORACLE_DB_USER or not ORACLE_DB_PASSWORD or not ORACLE_DB_DSN:
raise ValueError( raise ValueError(
"Oracle23ai requires setting ORACLE_DB_USER, ORACLE_DB_PASSWORD, and ORACLE_DB_DSN." "Oracle23ai requires setting ORACLE_DB_USER, ORACLE_DB_PASSWORD, and ORACLE_DB_DSN."
) )
if VECTOR_DB == "oracle23ai" and ORACLE_DB_USE_WALLET and (not ORACLE_WALLET_DIR or not ORACLE_WALLET_PASSWORD):
if VECTOR_DB == "oracle23ai" and ORACLE_DB_USE_WALLET and not ORACLE_WALLET_DIR or not ORACLE_WALLET_PASSWORD:
raise ValueError( raise ValueError(
"Oracle23ai requires setting ORACLE_WALLET_DIR and ORACLE_WALLET_PASSWORD when using wallet authentication." "Oracle23ai requires setting ORACLE_WALLET_DIR and ORACLE_WALLET_PASSWORD when using wallet authentication."
) )

229
backend/open_webui/retrieval/vector/dbs/oracle23ai.py
View File

@@ -1,6 +1,32 @@
"""
# ORACLE23AI (Oracle23ai Vector Search) : env.examples
VECTOR_DB = "oracle23ai"
## DBCS or oracle 23ai free
ORACLE_DB_USE_WALLET = false
ORACLE_DB_USER = "DEMOUSER"
ORACLE_DB_PASSWORD = "Welcome123456"
ORACLE_DB_DSN = "localhost:1521/FREEPDB1"
## ADW or ATP
# ORACLE_DB_USE_WALLET = true
# ORACLE_DB_USER = "DEMOUSER"
# ORACLE_DB_PASSWORD = "Welcome123456"
# ORACLE_DB_DSN = "medium"
# ORACLE_DB_DSN = "(description= (retry_count=3)(retry_delay=3)(address=(protocol=tcps)(port=1522)(host=xx.oraclecloud.com))(connect_data=(service_name=yy.adb.oraclecloud.com))(security=(ssl_server_dn_match=no)))"
# ORACLE_WALLET_DIR = "/home/opc/adb_wallet"
# ORACLE_WALLET_PASSWORD = "Welcome1"
ORACLE_VECTOR_LENGTH = 768
ORACLE_DB_POOL_MIN = 2
ORACLE_DB_POOL_MAX = 10
ORACLE_DB_POOL_INCREMENT = 1
"""
from typing import Optional, List, Dict, Any from typing import Optional, List, Dict, Any
from decimal import Decimal from decimal import Decimal
import logging
import os import os
import oracledb import oracledb
@@ -12,14 +38,27 @@ from open_webui.retrieval.vector.main import (
) )
from open_webui.config import ( from open_webui.config import (
ORACLE_DB_USE_WALLET ORACLE_DB_USE_WALLET,
ORACLE_DB_USER, ORACLE_DB_USER,
ORACLE_DB_PASSWORD, ORACLE_DB_PASSWORD,
ORACLE_DB_DSN, ORACLE_DB_DSN,
ORACLE_WALLET_DIR, ORACLE_WALLET_DIR,
ORACLE_WALLET_PASSWORD, ORACLE_WALLET_PASSWORD,
ORACLE_VECTOR_LENGTH, ORACLE_VECTOR_LENGTH,
ORACLE_DB_POOL_MIN,
ORACLE_DB_POOL_MAX,
ORACLE_DB_POOL_INCREMENT,
) )
from open_webui.env import SRC_LOG_LEVELS
log = logging.getLogger(__name__)
log.setLevel(SRC_LOG_LEVELS["RAG"])
# ORACLE_DB_USE_WALLET = os.environ.get("ORACLE_DB_USE_WALLET", "DBCS")
# ORACLE_DB_USER = os.environ.get("ORACLE_DB_USER", "DEMOUSER")
# ORACLE_DB_PASSWORD = os.environ.get("ORACLE_DB_PASSWORD", "Welcome123456")
# ORACLE_DB_DSN = os.environ.get("ORACLE_DB_DSN", "medium")
# ORACLE_DB_DSN = os.environ.get("ORACLE_DB_DSN", "(description= (retry_count=3)(retry_delay=3)(address=(protocol=tcps)(port=1522)(host=d6aqmjs6.adb.us-chicago-1.oraclecloud.com))(connect_data=(service_name=g13fc7c96b5ee55_agentvs_medium.adb.oraclecloud.com))(security=(ssl_server_dn_match=no)))")
class Oracle23aiClient(VectorDBBase): class Oracle23aiClient(VectorDBBase):
""" """
@@ -40,48 +79,145 @@ class Oracle23aiClient(VectorDBBase):
Creates a connection pool with min=2 and max=10 connections, initializes Creates a connection pool with min=2 and max=10 connections, initializes
the database schema if needed, and sets up necessary tables and indexes. the database schema if needed, and sets up necessary tables and indexes.
Args:
db_type (str): Database type - "ADB" for Autonomous Database or "DBCS" for Database Cloud Service
Raises: Raises:
ValueError: If required configuration parameters are missing ValueError: If required configuration parameters are missing
Exception: If database initialization fails Exception: If database initialization fails
""" """
try: try:
if not ORACLE_DB_DSN: # Create the appropriate connection pool based on DB type
raise ValueError("ORACLE_DB_DSN is required for Oracle Vector Search") if ORACLE_DB_USE_WALLET:
self._create_adb_pool()
self.pool = oracledb.create_pool( else: # DBCS
user=ORACLE_DB_USER, self._create_dbcs_pool()
password=ORACLE_DB_PASSWORD,
dsn=ORACLE_DB_DSN,
min=2,
max=10,
increment=1,
config_dir=ORACLE_WALLET_DIR,
wallet_location=ORACLE_WALLET_DIR,
wallet_password=ORACLE_WALLET_PASSWORD
)
log.info(f" >>> Creating Connection Pool [{ORACLE_DB_USER}:**@{ORACLE_DB_DSN}]") dsn = ORACLE_DB_DSN
log.info(f" >>> Creating Connection Pool [{ORACLE_DB_USER}:**@{dsn}]")
with self.get_connection() as connection: with self.get_connection() as connection:
log.info("Connection version:", connection.version) log.info("Connection version:", connection.version)
self._initialize_database(connection) self._initialize_database(connection)
print("Oracle Vector Search initialization complete.") log.info("Oracle Vector Search initialization complete.")
except Exception as e: except Exception as e:
print(f"Error during Oracle Vector Search initialization: {e}") log.exception(f"Error during Oracle Vector Search initialization: {e}")
raise raise
def _create_adb_pool(self) -> None:
"""
Create connection pool for Oracle Autonomous Database.
Uses wallet-based authentication.
"""
self.pool = oracledb.create_pool(
user=ORACLE_DB_USER,
password=ORACLE_DB_PASSWORD,
dsn=ORACLE_DB_DSN,
min=ORACLE_DB_POOL_MIN,
max=ORACLE_DB_POOL_MAX,
increment=ORACLE_DB_POOL_INCREMENT,
config_dir=ORACLE_WALLET_DIR,
wallet_location=ORACLE_WALLET_DIR,
wallet_password=ORACLE_WALLET_PASSWORD
)
log.info(f"Created ADB connection pool with wallet authentication.")
def _create_dbcs_pool(self) -> None:
"""
Create connection pool for Oracle Database Cloud Service.
Uses basic authentication without wallet.
"""
self.pool = oracledb.create_pool(
user=ORACLE_DB_USER,
password=ORACLE_DB_PASSWORD,
dsn=ORACLE_DB_DSN,
min=ORACLE_DB_POOL_MIN,
max=ORACLE_DB_POOL_MAX,
increment=ORACLE_DB_POOL_INCREMENT
)
log.info("Created DB connection pool with basic authentication.")
def get_connection(self): def get_connection(self):
""" """
Acquire a connection from the connection pool. Acquire a connection from the connection pool with retry logic.
Returns: Returns:
connection: A database connection with output type handler configured connection: A database connection with output type handler configured
""" """
connection = self.pool.acquire() max_retries = 3
connection.outputtypehandler = self._output_type_handler for attempt in range(max_retries):
return connection try:
connection = self.pool.acquire()
connection.outputtypehandler = self._output_type_handler
return connection
except oracledb.DatabaseError as e:
error_obj, = e.args
log.exception(f"Connection attempt {attempt + 1} failed: {error_obj.message}")
if attempt < max_retries - 1:
import time
wait_time = 2 ** attempt
log.info(f"Retrying in {wait_time} seconds...")
time.sleep(wait_time)
else:
raise
def start_health_monitor(self, interval_seconds: int = 60):
"""
Start a background thread to periodically check the health of the connection pool.
Args:
interval_seconds (int): Number of seconds between health checks
"""
def _monitor():
while True:
try:
log.info("[HealthCheck] Running periodic DB health check...")
self.ensure_connection()
log.info("[HealthCheck] Connection is healthy.")
except Exception as e:
log.exception(f"[HealthCheck] Connection health check failed: {e}")
time.sleep(interval_seconds)
thread = threading.Thread(target=_monitor, daemon=True)
thread.start()
log.info(f"Started DB health monitor every {interval_seconds} seconds.")
def _reconnect_pool(self):
"""
Attempt to reinitialize the connection pool if it's been closed or broken.
Args:
db_type (str): Database type - "ADB" for Autonomous Database or "DBCS" for Database Cloud Service
"""
try:
log.info("Attempting to reinitialize the Oracle connection pool...")
# Re-create the appropriate connection pool based on DB type
if ORACLE_DB_USE_WALLET:
self._create_adb_pool()
else: # DBCS
self._create_dbcs_pool()
log.info("Connection pool reinitialized.")
except Exception as e:
log.exception(f"Failed to reinitialize the connection pool: {e}")
raise
def ensure_connection(self):
"""
Ensure the database connection is alive, reconnecting pool if needed.
"""
try:
with self.get_connection() as connection:
with connection.cursor() as cursor:
cursor.execute("SELECT 1 FROM dual")
except Exception as e:
log.exception(f"Connection check failed: {e}, attempting to reconnect pool...")
self._reconnect_pool()
def _output_type_handler(self, cursor, metadata): def _output_type_handler(self, cursor, metadata):
""" """
Handle Oracle vector type conversion. Handle Oracle vector type conversion.
@@ -97,6 +233,8 @@ class Oracle23aiClient(VectorDBBase):
return cursor.var(metadata.type_code, arraysize=cursor.arraysize, return cursor.var(metadata.type_code, arraysize=cursor.arraysize,
outconverter=list) outconverter=list)
# Rest of the Oracle23aiClient class remains unchanged...
def _initialize_database(self, connection) -> None: def _initialize_database(self, connection) -> None:
""" """
Initialize database schema, tables and indexes. Initialize database schema, tables and indexes.
@@ -109,8 +247,9 @@ class Oracle23aiClient(VectorDBBase):
Raises: Raises:
Exception: If schema initialization fails Exception: If schema initialization fails
""" """
with connection.cursor() as cursor: with connection.cursor() as cursor:
print(f" >>> Creating Table document_chunk") log.info(f" >>> Creating Table document_chunk")
cursor.execute(f""" cursor.execute(f"""
BEGIN BEGIN
EXECUTE IMMEDIATE ' EXECUTE IMMEDIATE '
@@ -130,7 +269,7 @@ class Oracle23aiClient(VectorDBBase):
END; END;
""") """)
print(f" >>> Creating Table document_chunk_collection_name_idx") log.info(f" >>> Creating Table document_chunk_collection_name_idx")
cursor.execute(""" cursor.execute("""
BEGIN BEGIN
EXECUTE IMMEDIATE ' EXECUTE IMMEDIATE '
@@ -145,7 +284,7 @@ class Oracle23aiClient(VectorDBBase):
END; END;
""") """)
print(f" >>> Creating VECTOR INDEX document_chunk_vector_ivf_idx") log.info(f" >>> Creating VECTOR INDEX document_chunk_vector_ivf_idx")
cursor.execute(""" cursor.execute("""
BEGIN BEGIN
EXECUTE IMMEDIATE ' EXECUTE IMMEDIATE '
@@ -261,7 +400,7 @@ class Oracle23aiClient(VectorDBBase):
... ] ... ]
>>> client.insert("my_collection", items) >>> client.insert("my_collection", items)
""" """
print(f"Oracle23aiClient:Inserting {len(items)} items into collection '{collection_name}'.") log.info(f"Oracle23aiClient:Inserting {len(items)} items into collection '{collection_name}'.")
with self.get_connection() as connection: with self.get_connection() as connection:
try: try:
with connection.cursor() as cursor: with connection.cursor() as cursor:
@@ -282,10 +421,10 @@ class Oracle23aiClient(VectorDBBase):
}) })
connection.commit() connection.commit()
print(f"Oracle23aiClient:Inserted {len(items)} items into collection '{collection_name}'.") log.info(f"Oracle23aiClient:Inserted {len(items)} items into collection '{collection_name}'.")
except Exception as e: except Exception as e:
connection.rollback() connection.rollback()
print(f"Error during insert: {e}") log.exception(f"Error during insert: {e}")
raise raise
def upsert(self, collection_name: str, items: List[VectorItem]) -> None: def upsert(self, collection_name: str, items: List[VectorItem]) -> None:
@@ -344,10 +483,10 @@ class Oracle23aiClient(VectorDBBase):
}) })
connection.commit() connection.commit()
print(f"Upserted {len(items)} items into collection '{collection_name}'.") log.info(f"Upserted {len(items)} items into collection '{collection_name}'.")
except Exception as e: except Exception as e:
connection.rollback() connection.rollback()
print(f"Error during upsert: {e}") log.exception(f"Error during upsert: {e}")
raise raise
def search( def search(
@@ -425,9 +564,9 @@ class Oracle23aiClient(VectorDBBase):
metadatas=metadatas metadatas=metadatas
) )
except Exception as e: except Exception as e:
print(f"Error during search: {e}") log.exception(f"Error during search: {e}")
import traceback import traceback
print(traceback.format_exc()) log.exception(traceback.format_exc())
return None return None
def query( def query(
@@ -494,9 +633,9 @@ class Oracle23aiClient(VectorDBBase):
metadatas=metadatas metadatas=metadatas
) )
except Exception as e: except Exception as e:
print(f"Error during query: {e}") log.exception(f"Error during query: {e}")
import traceback import traceback
print(traceback.format_exc()) log.exception(traceback.format_exc())
return None return None
def get( def get(
@@ -552,9 +691,9 @@ class Oracle23aiClient(VectorDBBase):
metadatas=metadatas metadatas=metadatas
) )
except Exception as e: except Exception as e:
print(f"Error during get: {e}") log.exception(f"Error during get: {e}")
import traceback import traceback
print(traceback.format_exc()) log.exception(traceback.format_exc())
return None return None
def delete( def delete(
@@ -603,9 +742,9 @@ class Oracle23aiClient(VectorDBBase):
deleted = cursor.rowcount deleted = cursor.rowcount
connection.commit() connection.commit()
print(f"Deleted {deleted} items from collection '{collection_name}'.") log.info(f"Deleted {deleted} items from collection '{collection_name}'.")
except Exception as e: except Exception as e:
print(f"Error during delete: {e}") log.exception(f"Error during delete: {e}")
raise raise
def reset(self) -> None: def reset(self) -> None:
@@ -627,9 +766,9 @@ class Oracle23aiClient(VectorDBBase):
cursor.execute("DELETE FROM document_chunk") cursor.execute("DELETE FROM document_chunk")
deleted = cursor.rowcount deleted = cursor.rowcount
connection.commit() connection.commit()
print(f"Reset complete. Deleted {deleted} items from 'document_chunk' table.") log.info(f"Reset complete. Deleted {deleted} items from 'document_chunk' table.")
except Exception as e: except Exception as e:
print(f"Error during reset: {e}") log.exception(f"Error during reset: {e}")
raise raise
def close(self) -> None: def close(self) -> None:
@@ -680,7 +819,7 @@ class Oracle23aiClient(VectorDBBase):
count = cursor.fetchone()[0] count = cursor.fetchone()[0]
return count > 0 return count > 0
except Exception as e: except Exception as e:
print(f"Error checking collection existence: {e}") log.exception(f"Error checking collection existence: {e}")
return False return False
def delete_collection(self, collection_name: str) -> None: def delete_collection(self, collection_name: str) -> None:
@@ -697,4 +836,4 @@ class Oracle23aiClient(VectorDBBase):
>>> client.delete_collection("obsolete_collection") >>> client.delete_collection("obsolete_collection")
""" """
self.delete(collection_name) self.delete(collection_name)
print(f"Collection '{collection_name}' deleted.") log.info(f"Collection '{collection_name}' deleted.")

22
package-lock.json generated
View File

@@ -53,6 +53,7 @@
"jspdf": "^3.0.0", "jspdf": "^3.0.0",
"katex": "^0.16.22", "katex": "^0.16.22",
"kokoro-js": "^1.1.1", "kokoro-js": "^1.1.1",
"lowlight": "^3.3.0",
"marked": "^9.1.0", "marked": "^9.1.0",
"mermaid": "^11.6.0", "mermaid": "^11.6.0",
"paneforge": "^0.0.6", "paneforge": "^0.0.6",
@@ -3608,7 +3609,6 @@
"resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz", "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz",
"integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==", "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==",
"license": "MIT", "license": "MIT",
"peer": true,
"dependencies": { "dependencies": {
"@types/unist": "*" "@types/unist": "*"
} }
@@ -3692,8 +3692,7 @@
"node_modules/@types/unist": { "node_modules/@types/unist": {
"version": "2.0.10", "version": "2.0.10",
"resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.10.tgz", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.10.tgz",
"integrity": "sha512-IfYcSBWE3hLpBg8+X2SEa8LVkJdJEkT2Ese2aaLs3ptGdVtABxndrMaxuFlQ1qdFf9Q5rDvDpxI3WwgvKFAsQA==", "integrity": "sha512-IfYcSBWE3hLpBg8+X2SEa8LVkJdJEkT2Ese2aaLs3ptGdVtABxndrMaxuFlQ1qdFf9Q5rDvDpxI3WwgvKFAsQA=="
"peer": true
}, },
"node_modules/@types/yauzl": { "node_modules/@types/yauzl": {
"version": "2.10.3", "version": "2.10.3",
@@ -6055,7 +6054,6 @@
"resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz",
"integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==",
"license": "MIT", "license": "MIT",
"peer": true,
"dependencies": { "dependencies": {
"dequal": "^2.0.0" "dequal": "^2.0.0"
}, },
@@ -7336,9 +7334,10 @@
"dev": true "dev": true
}, },
"node_modules/highlight.js": { "node_modules/highlight.js": {
"version": "11.9.0", "version": "11.11.1",
"resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-11.9.0.tgz", "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-11.11.1.tgz",
"integrity": "sha512-fJ7cW7fQGCYAkgv4CPfwFHrfd/cLS4Hau96JuJ+ZTOWhjnhoeN1ub1tFmALm/+lW5z4WCAuAV9bm05AP0mS6Gw==", "integrity": "sha512-Xwwo44whKBVCYoliBQwaPvtd/2tYFkRQtXDWj1nackaV2JPXx3L0+Jvd8/qCJ2p+ML0/XVkJ2q+Mr+UVdpJK5w==",
"license": "BSD-3-Clause",
"engines": { "engines": {
"node": ">=12.0.0" "node": ">=12.0.0"
} }
@@ -8569,15 +8568,14 @@
} }
}, },
"node_modules/lowlight": { "node_modules/lowlight": {
"version": "3.1.0", "version": "3.3.0",
"resolved": "https://registry.npmjs.org/lowlight/-/lowlight-3.1.0.tgz", "resolved": "https://registry.npmjs.org/lowlight/-/lowlight-3.3.0.tgz",
"integrity": "sha512-CEbNVoSikAxwDMDPjXlqlFYiZLkDJHwyGu/MfOsJnF3d7f3tds5J3z8s/l9TMXhzfsJCCJEAsD78842mwmg0PQ==", "integrity": "sha512-0JNhgFoPvP6U6lE/UdVsSq99tn6DhjjpAj5MxG49ewd2mOBVtwWYIT8ClyABhq198aXXODMU6Ox8DrGy/CpTZQ==",
"license": "MIT", "license": "MIT",
"peer": true,
"dependencies": { "dependencies": {
"@types/hast": "^3.0.0", "@types/hast": "^3.0.0",
"devlop": "^1.0.0", "devlop": "^1.0.0",
"highlight.js": "~11.9.0" "highlight.js": "~11.11.0"
}, },
"funding": { "funding": {
"type": "github", "type": "github",

1
package.json
View File

@@ -97,6 +97,7 @@
"jspdf": "^3.0.0", "jspdf": "^3.0.0",
"katex": "^0.16.22", "katex": "^0.16.22",
"kokoro-js": "^1.1.1", "kokoro-js": "^1.1.1",
"lowlight": "^3.3.0",
"marked": "^9.1.0", "marked": "^9.1.0",
"mermaid": "^11.6.0", "mermaid": "^11.6.0",
"paneforge": "^0.0.6", "paneforge": "^0.0.6",

30
pyproject.toml
View File

@@ -10,18 +10,15 @@ dependencies = [
"uvicorn[standard]==0.34.2", "uvicorn[standard]==0.34.2",
"pydantic==2.10.6", "pydantic==2.10.6",
"python-multipart==0.0.20", "python-multipart==0.0.20",
"python-socketio==5.13.0", "python-socketio==5.13.0",
"python-jose==3.4.0", "python-jose==3.4.0",
"passlib[bcrypt]==1.7.4", "passlib[bcrypt]==1.7.4",
"requests==2.32.4", "requests==2.32.4",
"aiohttp==3.11.11", "aiohttp==3.11.11",
"async-timeout", "async-timeout",
"aiocache", "aiocache",
"aiofiles", "aiofiles",
"starlette-compress==1.6.0", "starlette-compress==1.6.0",
"sqlalchemy==2.0.38", "sqlalchemy==2.0.38",
"alembic==1.14.0", "alembic==1.14.0",
"peewee==3.18.1", "peewee==3.18.1",
@@ -30,29 +27,21 @@ dependencies = [
"pgvector==0.4.0", "pgvector==0.4.0",
"PyMySQL==1.1.1", "PyMySQL==1.1.1",
"bcrypt==4.3.0", "bcrypt==4.3.0",
"pymongo", "pymongo",
"redis", "redis",
"boto3==1.35.53", "boto3==1.35.53",
"argon2-cffi==23.1.0", "argon2-cffi==23.1.0",
"APScheduler==3.10.4", "APScheduler==3.10.4",
"RestrictedPython==8.0", "RestrictedPython==8.0",
"loguru==0.7.3", "loguru==0.7.3",
"asgiref==3.8.1", "asgiref==3.8.1",
"openai", "openai",
"anthropic", "anthropic",
"google-genai==1.15.0", "google-genai==1.15.0",
"google-generativeai==0.8.5", "google-generativeai==0.8.5",
"tiktoken", "tiktoken",
"langchain==0.3.24", "langchain==0.3.24",
"langchain-community==0.3.23", "langchain-community==0.3.23",
"fake-useragent==2.1.0", "fake-useragent==2.1.0",
"chromadb==0.6.3", "chromadb==0.6.3",
"pymilvus==2.5.0", "pymilvus==2.5.0",
@@ -61,13 +50,11 @@ dependencies = [
"playwright==1.49.1", "playwright==1.49.1",
"elasticsearch==9.0.1", "elasticsearch==9.0.1",
"pinecone==6.0.2", "pinecone==6.0.2",
"transformers", "transformers",
"sentence-transformers==4.1.0", "sentence-transformers==4.1.0",
"accelerate", "accelerate",
"colbert-ai==0.2.21", "colbert-ai==0.2.21",
"einops==0.8.1", "einops==0.8.1",
"ftfy==6.2.3", "ftfy==6.2.3",
"pypdf==4.3.1", "pypdf==4.3.1",
"fpdf2==2.8.2", "fpdf2==2.8.2",
@@ -87,51 +74,36 @@ dependencies = [
"sentencepiece", "sentencepiece",
"soundfile==0.13.1", "soundfile==0.13.1",
"azure-ai-documentintelligence==1.0.2", "azure-ai-documentintelligence==1.0.2",
"pillow==11.2.1", "pillow==11.2.1",
"opencv-python-headless==4.11.0.86", "opencv-python-headless==4.11.0.86",
"rapidocr-onnxruntime==1.4.4", "rapidocr-onnxruntime==1.4.4",
"rank-bm25==0.2.2", "rank-bm25==0.2.2",
"onnxruntime==1.20.1", "onnxruntime==1.20.1",
"faster-whisper==1.1.1", "faster-whisper==1.1.1",
"PyJWT[crypto]==2.10.1", "PyJWT[crypto]==2.10.1",
"authlib==1.4.1", "authlib==1.4.1",
"black==25.1.0", "black==25.1.0",
"langfuse==2.44.0", "langfuse==2.44.0",
"youtube-transcript-api==1.1.0", "youtube-transcript-api==1.1.0",
"pytube==15.0.0", "pytube==15.0.0",
"pydub", "pydub",
"duckduckgo-search==8.0.2", "duckduckgo-search==8.0.2",
"google-api-python-client", "google-api-python-client",
"google-auth-httplib2", "google-auth-httplib2",
"google-auth-oauthlib", "google-auth-oauthlib",
"docker~=7.1.0", "docker~=7.1.0",
"pytest~=8.3.2", "pytest~=8.3.2",
"pytest-docker~=3.1.1", "pytest-docker~=3.1.1",
"googleapis-common-protos==1.63.2", "googleapis-common-protos==1.63.2",
"google-cloud-storage==2.19.0", "google-cloud-storage==2.19.0",
"azure-identity==1.20.0", "azure-identity==1.20.0",
"azure-storage-blob==12.24.1", "azure-storage-blob==12.24.1",
"ldap3==2.9.1", "ldap3==2.9.1",
"firecrawl-py==1.12.0", "firecrawl-py==1.12.0",
"tencentcloud-sdk-python==3.0.1336", "tencentcloud-sdk-python==3.0.1336",
"gcp-storage-emulator>=2024.8.3", "gcp-storage-emulator>=2024.8.3",
"moto[s3]>=5.0.26", "moto[s3]>=5.0.26",
"oracledb>=3.2.0",
] ]
readme = "README.md" readme = "README.md"
requires-python = ">= 3.11, < 3.13.0a1" requires-python = ">= 3.11, < 3.13.0a1"

4647
uv.lock generated
View File

File diff suppressed because it is too large Load Diff

6909
yarn.lock Normal file
View File

File diff suppressed because it is too large Load Diff