Merge pull request #12683 from Youggls/dev

feat: Add sougou web search engine.

backend/open_webui/config.py

@@ -2142,6 +2142,18 @@ PERPLEXITY_API_KEY = PersistentConfig(
     os.getenv("PERPLEXITY_API_KEY", ""),
 )
 
+SOUGOU_API_SID = PersistentConfig(
+    "SOUGOU_API_SID",
+    "rag.web.search.sougou_api_sid",
+    os.getenv("SOUGOU_API_SID", ""),
+)
+
+SOUGOU_API_SK = PersistentConfig(
+    "SOUGOU_API_SK",
+    "rag.web.search.sougou_api_sk",
+    os.getenv("SOUGOU_API_SK", ""),
+)
+
 RAG_WEB_SEARCH_RESULT_COUNT = PersistentConfig(
     "RAG_WEB_SEARCH_RESULT_COUNT",
     "rag.web.search.result_count",

backend/open_webui/main.py

@@ -225,6 +225,8 @@ from open_webui.config import (
     BRAVE_SEARCH_API_KEY,
     EXA_API_KEY,
     PERPLEXITY_API_KEY,
+    SOUGOU_API_SID,
+    SOUGOU_API_SK,
     KAGI_SEARCH_API_KEY,
     MOJEEK_SEARCH_API_KEY,
     BOCHA_SEARCH_API_KEY,
@@ -652,6 +654,8 @@ app.state.config.BING_SEARCH_V7_ENDPOINT = BING_SEARCH_V7_ENDPOINT
 app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY = BING_SEARCH_V7_SUBSCRIPTION_KEY
 app.state.config.EXA_API_KEY = EXA_API_KEY
 app.state.config.PERPLEXITY_API_KEY = PERPLEXITY_API_KEY
+app.state.config.SOUGOU_API_SID = SOUGOU_API_SID
+app.state.config.SOUGOU_API_SK = SOUGOU_API_SK
 
 app.state.config.RAG_WEB_SEARCH_RESULT_COUNT = RAG_WEB_SEARCH_RESULT_COUNT
 app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS = RAG_WEB_SEARCH_CONCURRENT_REQUESTS

backend/open_webui/retrieval/web/sougou.py

@@ -0,0 +1,48 @@
+import logging
+import json
+from typing import Optional, List
+
+from tencentcloud.common.common_client import CommonClient
+from tencentcloud.common import credential
+from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException
+from tencentcloud.common.profile.client_profile import ClientProfile
+from tencentcloud.common.profile.http_profile import HttpProfile
+
+from open_webui.retrieval.web.main import SearchResult, get_filtered_results
+from open_webui.env import SRC_LOG_LEVELS
+
+log = logging.getLogger(__name__)
+log.setLevel(SRC_LOG_LEVELS["RAG"])
+
+
+def search_sougou(
+    sougou_api_sid: str,
+    sougou_api_sk: str,
+    query: str,
+    count: int,
+    filter_list: Optional[List[str]] = None,
+) -> List[SearchResult]:
+    try:
+        cred = credential.Credential(sougou_api_sid, sougou_api_sk)
+        http_profile = HttpProfile()
+        http_profile.endpoint = "tms.tencentcloudapi.com"
+        client_profile = ClientProfile()
+        client_profile.http_profile = http_profile
+        params = json.dumps({"Query": query, 'Cnt': 20})
+        common_client = CommonClient("tms", "2020-12-29", cred, "", profile=client_profile)
+        results = [
+            json.loads(page) for page in common_client.call_json("SearchPro", json.loads(params))["Response"]["Pages"]
+        ]
+        sorted_results = sorted(results, key=lambda x: x.get("scour", 0.0), reverse=True)
+        if filter_list:
+            sorted_results = get_filtered_results(sorted_results, filter_list)
+
+        return [
+            SearchResult(
+                link=result.get("url"), title=result.get("title"), snippet=result.get("passage")
+            )
+            for result in sorted_results[:count]
+        ]
+    except TencentCloudSDKException as err:
+        log.error(f"Error in Sougou search: {err}")
+        return []

backend/open_webui/routers/retrieval.py

@@ -60,6 +60,7 @@ from open_webui.retrieval.web.tavily import search_tavily
 from open_webui.retrieval.web.bing import search_bing
 from open_webui.retrieval.web.exa import search_exa
 from open_webui.retrieval.web.perplexity import search_perplexity
+from open_webui.retrieval.web.sougou import search_sougou
 
 from open_webui.retrieval.utils import (
     get_embedding_function,
@@ -411,6 +412,8 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
                 "bing_search_v7_subscription_key": request.app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY,
                 "exa_api_key": request.app.state.config.EXA_API_KEY,
                 "perplexity_api_key": request.app.state.config.PERPLEXITY_API_KEY,
+                "sougou_api_sid": request.app.state.config.SOUGOU_API_SID,
+                "sougou_api_sk": request.app.state.config.SOUGOU_API_SK,
                 "result_count": request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
                 "trust_env": request.app.state.config.RAG_WEB_SEARCH_TRUST_ENV,
                 "concurrent_requests": request.app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS,
@@ -478,6 +481,8 @@ class WebSearchConfig(BaseModel):
     bing_search_v7_subscription_key: Optional[str] = None
     exa_api_key: Optional[str] = None
     perplexity_api_key: Optional[str] = None
+    sougou_api_sid: Optional[str] = None
+    sougou_api_sk: Optional[str] = None
     result_count: Optional[int] = None
     concurrent_requests: Optional[int] = None
     trust_env: Optional[bool] = None
@@ -640,6 +645,12 @@ async def update_rag_config(
         request.app.state.config.PERPLEXITY_API_KEY = (
             form_data.web.search.perplexity_api_key
         )
+        request.app.state.config.SOUGOU_API_SID = (
+            form_data.web.search.sougou_api_sid
+        )
+        request.app.state.config.SOUGOU_API_SK = (
+            form_data.web.search.sougou_api_sk
+        )
 
         request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT = (
             form_data.web.search.result_count
@@ -712,6 +723,8 @@ async def update_rag_config(
                 "bing_search_v7_subscription_key": request.app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY,
                 "exa_api_key": request.app.state.config.EXA_API_KEY,
                 "perplexity_api_key": request.app.state.config.PERPLEXITY_API_KEY,
+                "sougou_api_sid": request.app.state.config.SOUGOU_API_SID,
+                "sougou_api_sk": request.app.state.config.SOUGOU_API_SK,
                 "result_count": request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
                 "concurrent_requests": request.app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS,
                 "trust_env": request.app.state.config.RAG_WEB_SEARCH_TRUST_ENV,
@@ -1267,6 +1280,7 @@ def search_web(request: Request, engine: str, query: str) -> list[SearchResult]:
     - TAVILY_API_KEY
     - EXA_API_KEY
     - PERPLEXITY_API_KEY
+    - SOUGOU_API_SID + SOUGOU_API_SK
     - SEARCHAPI_API_KEY + SEARCHAPI_ENGINE (by default `google`)
     - SERPAPI_API_KEY + SERPAPI_ENGINE (by default `google`)
     Args:
@@ -1438,6 +1452,17 @@ def search_web(request: Request, engine: str, query: str) -> list[SearchResult]:
             request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
             request.app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST,
         )
+    elif engine == 'sougou':
+        if request.app.state.config.SOUGOU_API_SID and request.app.state.config.SOUGOU_API_SK:
+            return search_sougou(
+                request.app.state.config.SOUGOU_API_SID,
+                request.app.state.config.SOUGOU_API_SK,
+                query,
+                request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
+                request.app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST,
+            )
+        else:
+            raise Exception("No SOUGOU_API_SID or SOUGOU_API_SK found in environment variables")
     else:
         raise Exception("No search engine API key found in environment variables")
 

backend/requirements.txt

@@ -123,6 +123,9 @@ ldap3==2.9.1
 ## Firecrawl
 firecrawl-py==1.12.0
 
+# Sougou API SDK(Tencentcloud SDK)
+tencentcloud-sdk-python==3.0.1336
+
 ## Trace
 opentelemetry-api==1.31.1
 opentelemetry-sdk==1.31.1