dev: add imported data as well

apiserver/plane/bgtasks/importer_task.py

@@ -1,29 +1,20 @@
 # Python imports
-import json
-import requests
 import uuid
 from functools import wraps
 
 # Django imports
 from django.db.models import Q, Max
-from django.conf import settings
-from django.core.serializers.json import DjangoJSONEncoder
 from django.contrib.auth.hashers import make_password
 
 # Third Party imports
 from celery import shared_task
-from sentry_sdk import capture_exception
 from celery.exceptions import MaxRetriesExceededError
 
 # Module imports
-from plane.app.serializers import ImporterSerializer
 from plane.db.models import (
     Importer,
     WorkspaceMember,
-    GithubRepositorySync,
-    GithubRepository,
     ProjectMember,
-    WorkspaceIntegration,
     Label,
     User,
     State,
@@ -33,7 +24,6 @@ from plane.db.models import (
     IssueProperty,
     IssueAssignee,
     IssueLabel,
-    Project,
     IssueSequence,
     IssueActivity,
     IssueComment,
@@ -50,26 +40,7 @@ def service_importer(service, importer_id):
     pass
 
 
-def handle_exceptions(task_func):
-    @wraps(task_func)
-    def wrapper(*args, **kwargs):
-        try:
-            return task_func(*args, **kwargs)
-        except Exception as e:
-            data = kwargs.get("data")
-            if data:
-                importer_id = data.get("importer_id")
-                status = data.get("status")
-                if importer_id and status:
-                    importer = Importer.objects.get(pk=importer_id)
-                    importer.status = status
-                    importer.reason = str(e)
-                    importer.save(update_fields=["status", "reason"])
-
-    return wrapper
-
-
-@handle_exceptions
+## Utility functions
 def get_label_id(name, data):
     try:
         existing_label = (
@@ -86,7 +57,6 @@ def get_label_id(name, data):
         return None
 
 
-@handle_exceptions
 def get_state_id(name, data):
     try:
         existing_state = (
@@ -103,7 +73,6 @@ def get_state_id(name, data):
         return None
 
 
-@handle_exceptions
 def get_user_id(name):
     try:
         existing_user = User.objects.filter(email=name).values("id").first()
@@ -112,9 +81,18 @@ def get_user_id(name):
         return None
 
 
+def update_imported_items(importer_id, entity, entity_id):
+    importer = Importer.objects.get(pk=importer_id)
+    if importer.imported_data:
+        importer.imported_data.setdefault(str(entity), []).append(str(entity_id))
+    else:
+        importer.imported_data = {
+            str(entity): [str(entity_id)]
+        }
+    importer.save()
+
+
 ## Sync functions
-
-
 def members_sync(data):
     try:
         user = User.objects.get(email=data.get("email"))
@@ -181,7 +159,7 @@ def label_sync(data):
     )
 
     if not existing_label.exists() and data.get("name"):
-        Label.objects.create(
+        label = Label.objects.create(
             project_id=data.get("project_id"),
             workspace_id=data.get("workspace_id"),
             name=data.get("name"),
@@ -190,6 +168,7 @@ def label_sync(data):
             external_id=data.get("external_id", None),
             external_source=data.get("external_source"),
         )
+        update_imported_items(data.get("importer_id"), "labels", label.id)
 
 
 def state_sync(data):
@@ -214,7 +193,7 @@ def state_sync(data):
             existing_state.external_source = data.get("external_source")
             existing_state.save()
         else:
-            State.objects.create(
+            state = State.objects.create(
                 project_id=data.get("project_id"),
                 workspace_id=data.get("workspace_id"),
                 name=data.get("state_name"),
@@ -223,6 +202,7 @@ def state_sync(data):
                 external_id=data.get("external_id"),
                 external_source=data.get("external_source"),
             )
+            update_imported_items(data.get("importer_id"), "states", state.id)
 
 
 def issue_sync(data):
@@ -276,7 +256,7 @@ def issue_sync(data):
             state_id=get_state_id(data.get("state"), data).get("id")
             if get_state_id(data.get("state"), data)
             else default_state.id,
-            name=data.get("name", "Issue Created through Importer"),
+            name=data.get("name", "Issue Created through Importer")[:255],
             description_html=data.get("description_html", "<p></p>"),
             sequence_id=last_id,
             sort_order=largest_sort_order,
@@ -318,6 +298,8 @@ def issue_sync(data):
             created_by_id=data.get("created_by_id"),
         )
 
+        update_imported_items(data.get("importer_id"), "issues", issue.id)
+
 
 def issue_label_sync(data):
     issue = Issue.objects.get(
@@ -385,7 +367,7 @@ def cycles_sync(data):
             workspace_id=data.get("workspace_id"),
         )
     except Cycle.DoesNotExist:
-        _ = Cycle.objects.create(
+        cycle = Cycle.objects.create(
             name=data.get("name"),
             description_html=data.get("description_html", "<p></p>"),
             project_id=data.get("project_id"),
@@ -394,6 +376,7 @@ def cycles_sync(data):
             external_id=data.get("external_id"),
             external_source=data.get("external_source"),
         )
+        update_imported_items(data.get("importer_id"), "cycles", cycle.id)
 
 
 def module_sync(data):
@@ -405,7 +388,7 @@ def module_sync(data):
             workspace_id=data.get("workspace_id"),
         )
     except Module.DoesNotExist:
-        _ = Module.objects.create(
+        module = Module.objects.create(
             name=data.get("name"),
             description_html=data.get("description_html", "<p></p>"),
             project_id=data.get("project_id"),
@@ -414,6 +397,7 @@ def module_sync(data):
             external_id=data.get("external_id"),
             external_source=data.get("external_source"),
         )
+        update_imported_items(data.get("importer_id"), "modules", module.id)
 
 
 def modules_issue_sync(data):
@@ -446,7 +430,6 @@ def import_sync(data):
 
 
 @shared_task(bind=True, queue="segway_task", max_retries=5)
-@handle_exceptions
 def import_task(self, data):
     type = data.get("type")
 
@@ -478,6 +461,10 @@ def import_task(self, data):
             # Retry with exponential backoff
             self.retry(exc=e, countdown=50, backoff=2)
         except MaxRetriesExceededError:
-            print(
-                f"Maximum retries reached for task. Exception: {e}, Type: {type}, Data: {data}"
-            )
+            # For max retries reached items fail the import
+            importer = Importer.objects.get(pk=data.get("importer_id"))
+            importer.status = "failed"
+            importer.reason = e
+            importer.save()
+
+        return

segway/src/controller/github.controller.ts

@@ -222,6 +222,7 @@ export class GithubController {
                 workspace_id: workspace_id,
                 project_id: project_id,
                 created_by_id: created_by,
+                importer_id: importer_id,
               },
             }, // kwargs
             other_data: {}, // other data
@@ -259,6 +260,7 @@ export class GithubController {
               workspace_id: workspace_id,
               project_id: project_id,
               created_by_id: created_by,
+              importer_id: importer_id,
             },
           }, // kwargs
           other_data: {}, // other data
@@ -323,6 +325,7 @@ export class GithubController {
                   url: issue.html_url,
                 },
                 parent_id: null,
+                importer_id: importer_id,
               },
             },
           };
@@ -349,6 +352,7 @@ export class GithubController {
                   workspace_id: workspace_id,
                   project_id: project_id,
                   created_by_id: created_by,
+                  importer_id: importer_id,
                 },
               },
             };
@@ -376,6 +380,7 @@ export class GithubController {
                   workspace_id: workspace_id,
                   project_id: project_id,
                   created_by_id: created_by,
+                  importer_id: importer_id,
                 },
               },
             };