diff --git a/.env.example b/.env.example index 71a9074a63..0649839a45 100644 --- a/.env.example +++ b/.env.example @@ -31,3 +31,5 @@ USE_MINIO=1 # Nginx Configuration NGINX_PORT=80 + +MONGO_DB_URL="mongodb://plane-mongodb:27017/" \ No newline at end of file diff --git a/apiserver/plane/bgtasks/api_logs_task.py b/apiserver/plane/bgtasks/api_logs_task.py index 038b939d54..13819cabf3 100644 --- a/apiserver/plane/bgtasks/api_logs_task.py +++ b/apiserver/plane/bgtasks/api_logs_task.py @@ -2,14 +2,69 @@ from django.utils import timezone from datetime import timedelta from plane.db.models import APIActivityLog from celery import shared_task +from django.conf import settings +from pymongo import MongoClient +from pymongo.errors import BulkWriteError +from plane.utils.exception_logger import log_exception + +BATCH_SIZE = 3000 @shared_task def delete_api_logs(): - # Get the logs older than 30 days to delete - logs_to_delete = APIActivityLog.objects.filter( - created_at__lte=timezone.now() - timedelta(days=30) - ) - # Delete the logs - logs_to_delete._raw_delete(logs_to_delete.db) + if settings.MONGO_DB_URL: + # Get the logs older than 30 days to delete + logs_to_delete = APIActivityLog.objects.filter( + created_at__lte=timezone.now() - timedelta(days=30) + ) + + # Create a MongoDB client + client = MongoClient(settings.MONGO_DB_URL) + db = client["plane"] + collection = db["api_activity_logs"] + + # Function to insert documents in batches + def bulk_insert(docs): + try: + collection.insert_many(docs) + except BulkWriteError as bwe: + log_exception(bwe) + + # Prepare the logs for bulk insert + def log_generator(): + batch = [] + for log in logs_to_delete.iterator(): + batch.append( + { + "token_identifier": log.token_identifier, + "path": log.path, + "method": log.method, + "query_params": log.query_params, + "headers": log.headers, + "body": log.body, + "response_body": log.response_body, + "response_code": log.response_code, + "ip_address": log.ip_address, + "user_agent": log.user_agent, + "created_at": log.created_at, + "updated_at": log.updated_at, + "created_by": str(log.created_by_id) if log.created_by_id else None, + "updated_by": str(log.updated_by_id) if log.updated_by_id else None, + } + ) + # If batch size is reached, yield the batch + if len(batch) == BATCH_SIZE: + yield batch + batch = [] + + # Yield the remaining logs + if batch: + yield batch + + # Upload the logs to MongoDB in batches + for batch in log_generator(): + bulk_insert(batch) + + # Delete the logs + logs_to_delete._raw_delete(logs_to_delete.db) diff --git a/apiserver/plane/db/mongodb.py b/apiserver/plane/db/mongodb.py new file mode 100644 index 0000000000..355e16f788 --- /dev/null +++ b/apiserver/plane/db/mongodb.py @@ -0,0 +1,23 @@ +from pymongo import MongoClient + +def singleton(cls): + instances = {} + + def wrapper(*args, **kwargs): + if cls not in instances: + instances[cls] = cls(*args, **kwargs) + return instances[cls] + + return wrapper + +@singleton +class Database: + db = None + client = None + + def __init__(self, mongo_uri, database_name): + self.client = MongoClient(mongo_uri) + self.db = self.client[database_name] + + def get_db(self): + return self.db diff --git a/apiserver/plane/middleware/api_log_middleware.py b/apiserver/plane/middleware/api_log_middleware.py index 96c62c2fd9..aab081dee6 100644 --- a/apiserver/plane/middleware/api_log_middleware.py +++ b/apiserver/plane/middleware/api_log_middleware.py @@ -38,6 +38,5 @@ class APITokenLogMiddleware: except Exception as e: print(e) - # If the token does not exist, you can decide whether to log this as an invalid attempt return None diff --git a/apiserver/plane/settings/common.py b/apiserver/plane/settings/common.py index b34dd17d59..a5e56542e5 100644 --- a/apiserver/plane/settings/common.py +++ b/apiserver/plane/settings/common.py @@ -350,3 +350,7 @@ INSTANCE_KEY = os.environ.get( SKIP_ENV_VAR = os.environ.get("SKIP_ENV_VAR", "1") == "1" DATA_UPLOAD_MAX_MEMORY_SIZE = int(os.environ.get("FILE_SIZE_LIMIT", 5242880)) + + +# MongoDB Settings +MONGO_DB_URL = os.environ.get("MONGO_DB_URL", False) diff --git a/apiserver/requirements/base.txt b/apiserver/requirements/base.txt index 2b7d383ba1..f728f093b6 100644 --- a/apiserver/requirements/base.txt +++ b/apiserver/requirements/base.txt @@ -34,4 +34,4 @@ posthog==3.0.2 cryptography==42.0.4 lxml==4.9.3 boto3==1.28.40 - +pymongo==4.6.1 \ No newline at end of file diff --git a/apiserver/templates/emails/auth/forgot_password.html b/apiserver/templates/emails/auth/forgot_password.html index a58a8cef75..9df90724f6 100644 --- a/apiserver/templates/emails/auth/forgot_password.html +++ b/apiserver/templates/emails/auth/forgot_password.html @@ -1,492 +1,47 @@ - - + + Set a new password to your Plane account - - - - - - + + + + + + - - + +