diff --git a/apiserver/plane/ee/documents/base.py b/apiserver/plane/ee/documents/base.py index 9cc2efa922..e06759200e 100644 --- a/apiserver/plane/ee/documents/base.py +++ b/apiserver/plane/ee/documents/base.py @@ -31,6 +31,16 @@ class BaseDocument(Document): settings = { "number_of_shards": settings.OPENSEARCH_SHARD_COUNT, "number_of_replicas": settings.OPENSEARCH_REPLICA_COUNT, + # Text search performance optimizations during heavy indexing + "refresh_interval": "30s", # Reduce refresh frequency (default: 1s) + # Indexing performance settings + "index.translog.flush_threshold_size": "1gb", # Larger translog before flush + "index.translog.sync_interval": "30s", # Less frequent syncing + # Search performance during indexing + "index.search.slowlog.threshold.query.warn": "1s", + "index.search.slowlog.threshold.query.info": "500ms", + "index.indexing.slowlog.threshold.index.warn": "5s", + "index.indexing.slowlog.threshold.index.info": "2s", "analysis": { "normalizer": { "lowercase_normalizer": lowercase_normalizer.get_definition() diff --git a/apiserver/plane/ee/utils/opensearch_helper.py b/apiserver/plane/ee/utils/opensearch_helper.py index f01dcc965a..094a595cda 100644 --- a/apiserver/plane/ee/utils/opensearch_helper.py +++ b/apiserver/plane/ee/utils/opensearch_helper.py @@ -10,7 +10,7 @@ from opensearchpy.exceptions import ( ConnectionError, NotFoundError, RequestError, - TransportError + TransportError, ) from opensearchpy.helpers.query import Q from rest_framework.serializers import Serializer @@ -66,7 +66,7 @@ class OpenSearchHelper: sort: Optional[List[str]] = None, operator: str = "or", result_key: Optional[str] = None, - serializer_class: Optional[Type[Serializer]] = None + serializer_class: Optional[Type[Serializer]] = None, ): """ Initialize the OpenSearch helper. @@ -91,7 +91,9 @@ class OpenSearchHelper: self.search_fields = search_fields or self._get_default_search_fields() self.source_fields = source_fields self.page = max(1, page) # Ensure page is at least 1 - self.page_size = min(page_size, getattr(settings, 'OPENSEARCH_MAX_PAGE_SIZE', 100)) + self.page_size = min( + page_size, getattr(settings, "OPENSEARCH_MAX_PAGE_SIZE", 100) + ) self.boosts = boosts or {} self.sort = sort self.operator = operator.lower() if operator else "or" @@ -105,8 +107,13 @@ class OpenSearchHelper: """Get default search fields based on the document class.""" # Prefer text fields with analyzers, especially edge_ngram_analyzer fields = [] - for field_name, field in self.document_cls._doc_type.mapping.properties.properties.to_dict().items(): - if field.get('type') == 'text': + for ( + field_name, + field, + ) in ( + self.document_cls._doc_type.mapping.properties.properties.to_dict().items() + ): + if field.get("type") == "text": fields.append(field_name) return fields @@ -118,13 +125,13 @@ class OpenSearchHelper: Q object with combined filters """ # Always filter out deleted items unless explicitly requested otherwise - has_deleted_filter = any('is_deleted' in f for f in self.filters) + has_deleted_filter = any("is_deleted" in f for f in self.filters) - filter_q = Q('bool', must=[Q('term', **f) for f in self.filters]) + filter_q = Q("bool", must=[Q("term", **f) for f in self.filters]) # Add default filters if not has_deleted_filter: - filter_q = filter_q & Q('term', is_deleted=False) + filter_q = filter_q & Q("term", is_deleted=False) return filter_q @@ -139,64 +146,58 @@ class OpenSearchHelper: return None # Categorize fields by their types for appropriate query construction - edge_ngram_fields = [] # Text fields with edge_ngram analyzer - standard_fields = [] # Regular text fields - keyword_fields = [] # Keyword fields - numeric_fields = [] # Integer, long, float, etc. + edge_ngram_fields = [] # Text fields with edge_ngram analyzer + standard_fields = [] # Regular text fields + keyword_fields = [] # Keyword fields + numeric_fields = [] # Integer, long, float, etc. for field_name in self.search_fields: field_info = self._field_meta(self.document_cls, field_name) if not field_info: continue - field_type = field_info.get('type', '') + field_type = field_info.get("type", "") # Handle text fields - if field_type == 'text': - analyzer = field_info.get('analyzer', '') - if 'edge_ngram' in analyzer: + if field_type == "text": + analyzer = field_info.get("analyzer", "") + if "edge_ngram" in analyzer: edge_ngram_fields.append(field_name) else: standard_fields.append(field_name) # Handle keyword fields - elif field_type == 'keyword': + elif field_type == "keyword": keyword_fields.append(field_name) # Handle numeric fields (integer, long, double, etc.) - elif field_type in ('integer', 'long', 'double', 'float'): + elif field_type in ("integer", "long", "double", "float"): numeric_fields.append(field_name) - # Apply boosts to fields - boosted_edge_ngram_fields = [f"{f}^{self.boosts.get(f, 1.0)}" for f in edge_ngram_fields] - boosted_standard_fields = [f"{f}^{self.boosts.get(f, 1.0)}" for f in standard_fields] + # Apply boosts to all text fields (edge_ngram + standard) + all_text_fields = edge_ngram_fields + standard_fields + boosted_text_fields = [ + f"{f}^{self.boosts.get(f, 1.0)}" for f in all_text_fields + ] # Build query components query_parts = [] - # Edge-ngram fields for prefix matching - if boosted_edge_ngram_fields: + # Combined text fields (edge_ngram + standard) for comprehensive matching + if boosted_text_fields: query_parts.append( - Q('multi_match', - query=self.query, - fields=boosted_edge_ngram_fields, - type='best_fields', - operator=self.operator) - ) - - # Standard fields for whole-word matching - if boosted_standard_fields: - query_parts.append( - Q('multi_match', - query=self.query, - fields=boosted_standard_fields, - type='best_fields', - operator=self.operator) + Q( + "multi_match", + query=self.query, + fields=boosted_text_fields, + type="best_fields", + operator=self.operator, + ) ) # Keyword fields for field in keyword_fields: boost = self.boosts.get(field, 1.0) query_parts.append( - Q('term', **{field: {"value": self.query, "boost": boost}}) + Q("term", **{field: {"value": self.query, "boost": boost}}) ) # Numeric fields - try to convert query to number if possible @@ -206,7 +207,7 @@ class OpenSearchHelper: boost = self.boosts.get(field, 1.0) # For numeric fields, use a term query with the converted value query_parts.append( - Q('term', **{field: {"value": numeric_value, "boost": boost}}) + Q("term", **{field: {"value": numeric_value, "boost": boost}}) ) except (ValueError, TypeError): # If query can't be converted to number, skip numeric fields @@ -218,7 +219,8 @@ class OpenSearchHelper: elif len(query_parts) == 1: return query_parts[0] else: - return Q('bool', should=query_parts) + # Use dis_max for better performance than bool should + return Q("dis_max", queries=query_parts, tie_breaker=0.3) def to_search(self) -> Search: """ @@ -242,9 +244,12 @@ class OpenSearchHelper: from_idx = (self.page - 1) * self.page_size search = search.extra(from_=from_idx, size=self.page_size) - # Apply source fields if provided + # Apply source fields if provided (reduces data transfer) if self.source_fields: - search = search.source(self.source_fields) + search = search.source(includes=self.source_fields) + else: + # For performance, exclude heavy fields by default + search = search.source(excludes=["description", "description_stripped"]) # Apply sorting if provided if self.sort: @@ -293,7 +298,7 @@ class OpenSearchHelper: raise @classmethod - def multi_search(cls, helpers: List['OpenSearchHelper']) -> MultiSearch: + def multi_search(cls, helpers: List["OpenSearchHelper"]) -> MultiSearch: """ Combine multiple helpers into a MultiSearch instance. @@ -312,8 +317,7 @@ class OpenSearchHelper: @classmethod def execute_multi_search( - cls, - helpers: List['OpenSearchHelper'] + cls, helpers: List["OpenSearchHelper"] ) -> Dict[str, List[Dict[str, Any]]]: """ Execute a multi-search query and organize results by each helper's result_key. @@ -362,8 +366,7 @@ class OpenSearchHelper: @staticmethod def serialize_hits( - hits: Sequence, - serializer_class: Type[Serializer] + hits: Sequence, serializer_class: Type[Serializer] ) -> List[Dict[str, Any]]: """ Serialize raw hits with DRF serializers. diff --git a/apiserver/plane/settings/common.py b/apiserver/plane/settings/common.py index eb1c9105ad..9e385e4de6 100644 --- a/apiserver/plane/settings/common.py +++ b/apiserver/plane/settings/common.py @@ -568,6 +568,22 @@ OPENSEARCH_ENABLED = os.environ.get("OPENSEARCH_ENABLED", "0") == "1" OPENSEARCH_INDEX_PREFIX = os.environ.get("OPENSEARCH_INDEX_PREFIX", "") OPENSEARCH_SHARD_COUNT = os.environ.get("OPENSEARCH_SHARD_COUNT", 1) OPENSEARCH_REPLICA_COUNT = os.environ.get("OPENSEARCH_REPLICA_COUNT", 0) + +# Text Search Performance Optimization +OPENSEARCH_SEARCH_TIMEOUT = int( + os.environ.get("OPENSEARCH_SEARCH_TIMEOUT", "60") +) # seconds +OPENSEARCH_MAX_PAGE_SIZE = int(os.environ.get("OPENSEARCH_MAX_PAGE_SIZE", "100")) +OPENSEARCH_DEFAULT_PAGE_SIZE = int(os.environ.get("OPENSEARCH_DEFAULT_PAGE_SIZE", "25")) + +# Optimizations for 2-active-data-node setup with heavy indexing +OPENSEARCH_BULK_CHUNK_SIZE = int( + os.environ.get("OPENSEARCH_BULK_CHUNK_SIZE", "500") +) # Smaller chunks +OPENSEARCH_INDEXING_TIMEOUT = int( + os.environ.get("OPENSEARCH_INDEXING_TIMEOUT", "120") +) # Longer indexing timeout + if OPENSEARCH_ENABLED: # OpenSearch Config OPENSEARCH_DSL = { @@ -580,7 +596,13 @@ if OPENSEARCH_ENABLED: "use_ssl": True, "verify_certs": False, "ssl_show_warn": False, - "timeout": 60, + "timeout": OPENSEARCH_SEARCH_TIMEOUT, + # Connection pool optimization for 2-data-node setup + "maxsize": 15, # Reduced from 25 to not overwhelm 2 data nodes + "max_retries": 3, + "retry_on_timeout": True, + # Bulk indexing optimizations + "http_compress": True, # Reduce network overhead } } OPENSEARCH_DSL_SIGNAL_PROCESSOR = os.environ.get(