diff --git a/apiserver/plane/bgtasks/importer_task.py b/apiserver/plane/bgtasks/importer_task.py
index c39e7cd35d..bdf387c60b 100644
--- a/apiserver/plane/bgtasks/importer_task.py
+++ b/apiserver/plane/bgtasks/importer_task.py
@@ -1,29 +1,20 @@
# Python imports
-import json
-import requests
import uuid
from functools import wraps
# Django imports
from django.db.models import Q, Max
-from django.conf import settings
-from django.core.serializers.json import DjangoJSONEncoder
from django.contrib.auth.hashers import make_password
# Third Party imports
from celery import shared_task
-from sentry_sdk import capture_exception
from celery.exceptions import MaxRetriesExceededError
# Module imports
-from plane.app.serializers import ImporterSerializer
from plane.db.models import (
Importer,
WorkspaceMember,
- GithubRepositorySync,
- GithubRepository,
ProjectMember,
- WorkspaceIntegration,
Label,
User,
State,
@@ -33,7 +24,6 @@ from plane.db.models import (
IssueProperty,
IssueAssignee,
IssueLabel,
- Project,
IssueSequence,
IssueActivity,
IssueComment,
@@ -50,26 +40,7 @@ def service_importer(service, importer_id):
pass
-def handle_exceptions(task_func):
- @wraps(task_func)
- def wrapper(*args, **kwargs):
- try:
- return task_func(*args, **kwargs)
- except Exception as e:
- data = kwargs.get("data")
- if data:
- importer_id = data.get("importer_id")
- status = data.get("status")
- if importer_id and status:
- importer = Importer.objects.get(pk=importer_id)
- importer.status = status
- importer.reason = str(e)
- importer.save(update_fields=["status", "reason"])
-
- return wrapper
-
-
-@handle_exceptions
+## Utility functions
def get_label_id(name, data):
try:
existing_label = (
@@ -86,7 +57,6 @@ def get_label_id(name, data):
return None
-@handle_exceptions
def get_state_id(name, data):
try:
existing_state = (
@@ -103,7 +73,6 @@ def get_state_id(name, data):
return None
-@handle_exceptions
def get_user_id(name):
try:
existing_user = User.objects.filter(email=name).values("id").first()
@@ -112,9 +81,18 @@ def get_user_id(name):
return None
+def update_imported_items(importer_id, entity, entity_id):
+ importer = Importer.objects.get(pk=importer_id)
+ if importer.imported_data:
+ importer.imported_data.setdefault(str(entity), []).append(str(entity_id))
+ else:
+ importer.imported_data = {
+ str(entity): [str(entity_id)]
+ }
+ importer.save()
+
+
## Sync functions
-
-
def members_sync(data):
try:
user = User.objects.get(email=data.get("email"))
@@ -181,7 +159,7 @@ def label_sync(data):
)
if not existing_label.exists() and data.get("name"):
- Label.objects.create(
+ label = Label.objects.create(
project_id=data.get("project_id"),
workspace_id=data.get("workspace_id"),
name=data.get("name"),
@@ -190,6 +168,7 @@ def label_sync(data):
external_id=data.get("external_id", None),
external_source=data.get("external_source"),
)
+ update_imported_items(data.get("importer_id"), "labels", label.id)
def state_sync(data):
@@ -214,7 +193,7 @@ def state_sync(data):
existing_state.external_source = data.get("external_source")
existing_state.save()
else:
- State.objects.create(
+ state = State.objects.create(
project_id=data.get("project_id"),
workspace_id=data.get("workspace_id"),
name=data.get("state_name"),
@@ -223,6 +202,7 @@ def state_sync(data):
external_id=data.get("external_id"),
external_source=data.get("external_source"),
)
+ update_imported_items(data.get("importer_id"), "states", state.id)
def issue_sync(data):
@@ -276,7 +256,7 @@ def issue_sync(data):
state_id=get_state_id(data.get("state"), data).get("id")
if get_state_id(data.get("state"), data)
else default_state.id,
- name=data.get("name", "Issue Created through Importer"),
+ name=data.get("name", "Issue Created through Importer")[:255],
description_html=data.get("description_html", ""),
sequence_id=last_id,
sort_order=largest_sort_order,
@@ -318,6 +298,8 @@ def issue_sync(data):
created_by_id=data.get("created_by_id"),
)
+ update_imported_items(data.get("importer_id"), "issues", issue.id)
+
def issue_label_sync(data):
issue = Issue.objects.get(
@@ -385,7 +367,7 @@ def cycles_sync(data):
workspace_id=data.get("workspace_id"),
)
except Cycle.DoesNotExist:
- _ = Cycle.objects.create(
+ cycle = Cycle.objects.create(
name=data.get("name"),
description_html=data.get("description_html", ""),
project_id=data.get("project_id"),
@@ -394,6 +376,7 @@ def cycles_sync(data):
external_id=data.get("external_id"),
external_source=data.get("external_source"),
)
+ update_imported_items(data.get("importer_id"), "cycles", cycle.id)
def module_sync(data):
@@ -405,7 +388,7 @@ def module_sync(data):
workspace_id=data.get("workspace_id"),
)
except Module.DoesNotExist:
- _ = Module.objects.create(
+ module = Module.objects.create(
name=data.get("name"),
description_html=data.get("description_html", ""),
project_id=data.get("project_id"),
@@ -414,6 +397,7 @@ def module_sync(data):
external_id=data.get("external_id"),
external_source=data.get("external_source"),
)
+ update_imported_items(data.get("importer_id"), "modules", module.id)
def modules_issue_sync(data):
@@ -446,7 +430,6 @@ def import_sync(data):
@shared_task(bind=True, queue="segway_task", max_retries=5)
-@handle_exceptions
def import_task(self, data):
type = data.get("type")
@@ -478,6 +461,10 @@ def import_task(self, data):
# Retry with exponential backoff
self.retry(exc=e, countdown=50, backoff=2)
except MaxRetriesExceededError:
- print(
- f"Maximum retries reached for task. Exception: {e}, Type: {type}, Data: {data}"
- )
+ # For max retries reached items fail the import
+ importer = Importer.objects.get(pk=data.get("importer_id"))
+ importer.status = "failed"
+ importer.reason = e
+ importer.save()
+
+ return
diff --git a/segway/src/controller/github.controller.ts b/segway/src/controller/github.controller.ts
index c637039c61..e395a73e53 100644
--- a/segway/src/controller/github.controller.ts
+++ b/segway/src/controller/github.controller.ts
@@ -222,6 +222,7 @@ export class GithubController {
workspace_id: workspace_id,
project_id: project_id,
created_by_id: created_by,
+ importer_id: importer_id,
},
}, // kwargs
other_data: {}, // other data
@@ -259,6 +260,7 @@ export class GithubController {
workspace_id: workspace_id,
project_id: project_id,
created_by_id: created_by,
+ importer_id: importer_id,
},
}, // kwargs
other_data: {}, // other data
@@ -323,6 +325,7 @@ export class GithubController {
url: issue.html_url,
},
parent_id: null,
+ importer_id: importer_id,
},
},
};
@@ -349,6 +352,7 @@ export class GithubController {
workspace_id: workspace_id,
project_id: project_id,
created_by_id: created_by,
+ importer_id: importer_id,
},
},
};
@@ -376,6 +380,7 @@ export class GithubController {
workspace_id: workspace_id,
project_id: project_id,
created_by_id: created_by,
+ importer_id: importer_id,
},
},
};