mirror of
https://github.com/makeplane/plane.git
synced 2026-09-03 12:50:29 +02:00
eef03e671e
* Pgtrigger and Outbox prototype * feat: add event stream app with models and migrations - Introduced the event_stream app with initial models: Outbox and IssueProxy. - Implemented a PostgreSQL trigger for logging updates to issues. - Added necessary migrations and updated settings to include the new app. * refactor: migrate event publisher to event stream - Updated imports in test files to reflect the new event_stream app structure. - Removed the event_publisher app, including its models, migrations, and tests. - Introduced MemorySafeOutboxEventListener and related service for handling outbox events. - Added management command for starting the outbox event listener with memory safety. - Implemented PostgreSQL triggers for outbox notifications in the new event_stream app. * refactor: enhance issue serializer and event stream models - Updated IssueSerializer and IssueCreateSerializer to manage assignees and labels more efficiently by calculating additions and removals instead of deleting all and re-adding. - Introduced new proxy models for CycleIssue and ModuleIssue with corresponding PostgreSQL triggers for event handling. - Refactored Outbox model to include entity type and ID for better event tracking. - Removed outdated IssueProxy model and its associated triggers, streamlining the event stream architecture. * chore: update base requirements for OpenSearch and add pika for RabbitMQ - Retained django-opensearch-dsl version 0.7.0 in requirements. - Added pika version 1.3.2 for RabbitMQ integration. * refactor: update import paths for event stream listener - Changed import statements in test_competitive_listeners.py, service.py, and listen_outbox_events.py to reflect the new module structure by removing the 'apiserver' prefix. * feat: implement PostgreSQL LISTEN/NOTIFY listener for outbox events - Added DatabaseConnection class to manage psycopg3 connections for LISTEN/NOTIFY. - Introduced NotificationListener class to handle PostgreSQL NOTIFY messages and dispatch them to registered handlers. - Updated Command class to utilize NotificationListener for listening to outbox events, replacing the previous restartable listener implementation. - Improved error handling and logging throughout the listener process. * feat: enhance event processing with advisory locks - Implemented advisory lock mechanism to prevent concurrent processing of events. - Added methods to acquire and release advisory locks in the DatabaseConnection class. - Updated root_handler to fetch complete outbox data and mark events as processed only if all handlers succeed. - Improved error handling and logging for event processing. * fix: improve event logging and error handling in NotificationListener - Updated event logging to include event ID for better traceability. - Enhanced error handling in the command's handle method to log exceptions and maintain robustness during event listening. * feat: add memory monitoring and auto-restart functionality to event listener - Introduced MemoryMonitor class to track memory usage and event processing counts. - Implemented automatic restarts for the NotificationListener based on memory and event limits. - Enhanced command line options to configure memory limits, event limits, and memory check intervals. - Improved logging for memory stats and event processing to aid in monitoring and debugging. * feat: update local and production settings for event logging - Introduced logging configuration for 'plane.event_stream' in both local and production settings to enhance event logging capabilities. * chore: Implement async outbox polling mechanism with memory monitoring and event handling. Add management command for outbox poller and create necessary database migrations for outbox triggers and indexes. * docs: Add comprehensive README for Event Stream system detailing architecture, components, configuration, usage, and performance tuning. * feat: Enhance MongoDB integration with a singleton connection manager and implement outbox cleanup task for migrating records from PostgreSQL to MongoDB * feat: Add asyncio support for tests and new test files for outbox polling * Merge branch 'event-stream' of github.com:makeplane/plane-ee into event-stream * refactor: Update event types in outbox triggers for cycle, issue, and module models; add IssueComment, IssueLink, and IssueRelation proxies with corresponding triggers * refactor: Update import paths for JsonFormatter and consolidate OpenFeature imports; add unit tests for OutboxPoller and related classes * refactor: Replace all_objects with objects in Outbox queries and update test fixtures for outbox records * feat: Add daily outbox cleaner task to Celery and update type hint for MongoDB collection * refactor: Enhance memory monitoring and outbox polling logic with improved logging and delay handling * feat: Implement outbox triggers for issue, issue attachment, issue link, and issue relation models with dynamic event type determination * chore: Remove obsolete files related to multiple listeners and competitive processing tests * Delete service.py * feat: Add outbox poller script to manage database migrations and process events with configurable parameters * refactor: Optimize EpicCreateSerializer to handle initiative, assignee, and label updates with conflict management and improved logic for adding/removing related entities * feat: Update issue proxy triggers for outbox events - Removed the existing 'issue_outbox_update' trigger from the IssueProxy model. - Added a new 'issue_outbox_update' trigger with enhanced logic to handle conversion events when the issue type changes. - The new trigger captures updates and soft deletes, inserting appropriate events into the outbox. * feat: Refactor issue proxy triggers and add new outbox event handling - Removed and replaced existing triggers for IssueProxy, IssueAssigneeProxy, IssueLabelProxy, CycleIssueProxy, and ModuleIssueProxy. - Introduced new triggers to handle outbox events for various issue-related actions, including creation, updates, and deletions. - Enhanced logic for event type determination based on issue type, ensuring accurate event handling in the outbox. * refactor: Update serializers to use ID fields for assignees and labels - Changed the handling of assignees and labels in IssueCreateSerializer and EpicCreateSerializer to use ID fields instead of object references. - Improved logic for determining assignees and labels to add or remove by using lists of IDs. - Enhanced code readability with comments explaining the changes. * feat: Enhance issue event handling in outbox triggers - Updated triggers for IssueProxy, IssueAssigneeProxy, and IssueLabelProxy to determine event types based on issue type, including handling for epics. - Introduced filtered data for outbox events to exclude description fields, improving data integrity and reducing unnecessary payload size. - Enhanced logic for detecting changes during updates, ensuring only relevant changes are captured and processed. * feat: implement connection pooling for outbox poller - Introduced DatabaseConnectionPool class to manage async connection pooling using psycopg_pool. - Enhanced outbox polling logic to utilize connection pooling for improved performance and resource management. - Added health check and statistics retrieval methods for the connection pool. - Updated outbox model and migration to include claimed_at field for better event tracking. - Refactored tests to validate new connection pooling functionality. * refactor: move MongoConnection to a new settings module and update imports - Created a new mongo.py file to define the MongoConnection class for managing MongoDB connections. - Updated the import path in outbox_cleaner.py to reference the new MongoConnection location. * refactor: update import path for MongoConnection in outbox_cleaner.py * chore: remove scout_apm from production settings and requirements - Deleted scout_apm from the installed apps in production.py. - Removed scout-apm dependency from base.txt requirements. * feat: add claimed_at field to outbox events and update requirements - Added claimed_at field to the outbox event model for enhanced event tracking. - Updated outbox poller to handle claimed_at in event processing. - Included django-pgtrigger in base.txt requirements for database trigger management. * feat: enhance memory monitoring in outbox poller - Updated MemoryMonitor to signal for restarts instead of exiting on memory limit exceedance. - Added methods to check for restart requests and wait for restart signals. - Improved outbox poller logic to handle memory monitoring more gracefully during processing. - Ensured all claimed rows are processed before initiating a restart. * feat: implement graceful shutdown handling in outbox poller - Added GracefulShutdownHandler class to manage shutdown signals (SIGTERM, SIGINT, SIGQUIT). - Integrated shutdown handling into the outbox poller to allow for graceful exits during processing. - Updated polling logic to check for shutdown requests and clean up resources accordingly. - Enhanced command help text to reflect new signal handling capabilities. * Enhance outbox event handling by adding workspace and project IDs - Updated the outbox cleaner to include `workspace_id` and `project_id` in the deletion process. - Modified the outbox poller to handle new fields in event processing. - Adjusted models and migrations to support the new fields in the outbox table. - Updated tests to ensure proper handling of workspace and project IDs in outbox records. * Merge branch 'preview' of github.com:makeplane/plane-ee into event-stream * feat: add initiator_id field to Outbox model and update migration dependencies * feat: add initiator_id to Outbox insertions across event stream models - Updated Outbox insert statements in various models to include initiator_id. - Adjusted related logic in CycleIssueProxy, IssueProxy, ModuleIssueProxy, and others to ensure proper tracking of the user initiating changes. - Enhanced event handling for soft deletes and regular updates to maintain consistency in data tracking. * feat: include initiator_id in OutboxEvent model and database interactions - Added initiator_id field to the OutboxEvent model for enhanced tracking. - Updated database queries in outbox_poller.py to include initiator_id in insertions. - Ensured consistency in data handling across event stream components. * refactor: update handle_row function to use OutboxEvent model - Changed the parameter type of handle_row from Dict to OutboxEvent for better type safety. - Updated logging to utilize the to_dict method of OutboxEvent for consistent event data representation. * feat: enhance Outbox model with initiator type and update related logic - Added initiator_type field to the Outbox model to track the type of event initiator. - Introduced InitiatorTypes enum for better clarity and management of initiator types. - Updated OutboxEvent model and related methods to include initiator_type for consistent event data representation. - Adjusted database interactions across event stream models to accommodate the new initiator_type field. * feat: add initiator_type to Outbox insertions across event stream models - Updated Outbox insert statements in various models to include initiator_type for better tracking of event initiators. - Adjusted related logic in CycleIssueProxy, IssueProxy, ModuleIssueProxy, and others to ensure consistent handling of initiator_type during event processing. - Enhanced event handling for both soft deletes and regular updates to maintain data integrity and tracking. * feat: update event stream triggers to use After timing for improved consistency - Changed trigger timing from Before to After for various event stream models including CycleIssueProxy, IssueProxy, and IssueAssigneeProxy to ensure that outbox updates reflect the final state of the entities. - Enhanced logic in triggers to include previous attributes for better tracking of changes during updates. - Adjusted related logic in IssueLabelProxy and IssueCommentProxy to maintain consistency across event handling. * feat: enhance bulk update task with transaction management and initiator type - Wrapped bulk creation of issue relations and parent ID updates in a transaction to ensure atomicity. - Set the initiator type to 'SYSTEM.IMPORT' for both bulk creation and updates to improve tracking of operations. - Improved database interaction by using a connection cursor for executing SQL commands. --------- Co-authored-by: Dheeraj Kumar Ketireddy <dheeru0198@gmail.com>
83 lines
2.1 KiB
Python
83 lines
2.1 KiB
Python
"""Development settings"""
|
|
|
|
import os
|
|
|
|
from .common import * # noqa
|
|
|
|
DEBUG = True
|
|
|
|
# Debug Toolbar settings
|
|
INSTALLED_APPS += ("debug_toolbar",) # noqa
|
|
MIDDLEWARE += ("debug_toolbar.middleware.DebugToolbarMiddleware",) # noqa
|
|
|
|
DEBUG_TOOLBAR_PATCH_SETTINGS = False
|
|
|
|
# Only show emails in console don't send it to smtp
|
|
EMAIL_BACKEND = os.environ.get(
|
|
"EMAIL_BACKEND", "django.core.mail.backends.console.EmailBackend"
|
|
)
|
|
|
|
CACHES = {
|
|
"default": {
|
|
"BACKEND": "django_redis.cache.RedisCache",
|
|
"LOCATION": REDIS_URL, # noqa
|
|
"OPTIONS": {"CLIENT_CLASS": "django_redis.client.DefaultClient"},
|
|
}
|
|
}
|
|
|
|
INTERNAL_IPS = ("127.0.0.1",)
|
|
|
|
MEDIA_URL = "/uploads/"
|
|
MEDIA_ROOT = os.path.join(BASE_DIR, "uploads") # noqa
|
|
|
|
LOG_DIR = os.path.join(BASE_DIR, "logs") # noqa
|
|
|
|
if not os.path.exists(LOG_DIR):
|
|
os.makedirs(LOG_DIR)
|
|
|
|
LOGGING = {
|
|
"version": 1,
|
|
"disable_existing_loggers": False,
|
|
"formatters": {
|
|
"verbose": {
|
|
"format": "{levelname} {asctime} {module} {process:d} {thread:d} {message}",
|
|
"style": "{",
|
|
},
|
|
"json": {
|
|
"()": "pythonjsonlogger.json.JsonFormatter",
|
|
"fmt": "%(levelname)s %(asctime)s %(module)s %(name)s %(message)s",
|
|
},
|
|
},
|
|
"handlers": {
|
|
"console": {
|
|
"level": "DEBUG",
|
|
"class": "logging.StreamHandler",
|
|
"formatter": "json",
|
|
}
|
|
},
|
|
"loggers": {
|
|
"plane.api.request": {
|
|
"level": "INFO",
|
|
"handlers": ["console"],
|
|
"propagate": False,
|
|
},
|
|
"plane.api": {"level": "INFO", "handlers": ["console"], "propagate": False},
|
|
"plane.worker": {"level": "INFO", "handlers": ["console"], "propagate": False},
|
|
"plane.exception": {
|
|
"level": "ERROR",
|
|
"handlers": ["console"],
|
|
"propagate": False,
|
|
},
|
|
"plane.external": {
|
|
"level": "INFO",
|
|
"handlers": ["console"],
|
|
"propagate": False,
|
|
},
|
|
"plane.event_stream": {
|
|
"level": "INFO",
|
|
"handlers": ["console"],
|
|
"propagate": False,
|
|
},
|
|
},
|
|
}
|