[SILO-347] Add retry logic in case of 502 for plane apis #3547

silo/src/etl/migrator/issue-types/issue-property.migrator.ts

@@ -31,14 +31,14 @@ export const createOrUpdateIssueProperties = async (
     props;
 
   // Process a single issue property
-  const processIssueProperty = async (issueProperty: Partial<ExIssueProperty>): Promise<ExIssueProperty | null> => {
+  const processIssueProperty = async (issueProperty: Partial<ExIssueProperty>): Promise<ExIssueProperty | undefined> => {
     try {
       const issueType = issueTypesMap.get(issueProperty.type_id || "") || defaultIssueType;
       if (!issueType) {
         logger.error(
           `[${jobId.slice(0, 7)}] Issue type not found for the issue property: ${issueProperty.display_name}`
         );
-        return null;
+        return undefined;
       }
 
       let createdUpdatedIssueProperty: ExIssueProperty | undefined;
@@ -68,14 +68,14 @@ export const createOrUpdateIssueProperties = async (
         `[${jobId.slice(0, 7)}] Error while ${method === "create" ? "creating" : "updating"} the issue property: ${issueProperty.display_name}`,
         error
       );
-      return null;
+      return undefined;
     }
   };
 
   // Process all issue properties in batches of 5
   const createdUpdatedIssueProperties = await processBatchPromises(issueProperties, processIssueProperty, 5);
 
-  return createdUpdatedIssueProperties;
+  return createdUpdatedIssueProperties.filter((property) => property !== undefined) as ExIssueProperty[];
 };
 
 export const createOrUpdateIssuePropertiesOptions = async (
@@ -85,7 +85,7 @@ export const createOrUpdateIssuePropertiesOptions = async (
 
   const processIssuePropertyOption = async (
     issuePropertyOption: Partial<ExIssuePropertyOption>
-  ): Promise<ExIssuePropertyOption | null> => {
+  ): Promise<ExIssuePropertyOption | undefined> => {
     let createdUpdatedIssuePropertyOption: ExIssuePropertyOption | undefined;
     try {
       const issueProperty = issuePropertyMap.get(issuePropertyOption.property_id || "");
@@ -93,7 +93,7 @@ export const createOrUpdateIssuePropertiesOptions = async (
         logger.error(
           `[${jobId.slice(0, 7)}] Issue property not found for the issue property option: ${issuePropertyOption.name}`
         );
-        return null;
+        return undefined;
       }
 
       if (method === "create") {
@@ -121,7 +121,7 @@ export const createOrUpdateIssuePropertiesOptions = async (
         `[${jobId.slice(0, 7)}] Error while ${method === "create" ? "creating" : "updating"} the issue property option: ${issuePropertyOption.name}`,
         error
       );
-      return null;
+      return undefined;
     }
   };
 
@@ -131,5 +131,5 @@ export const createOrUpdateIssuePropertiesOptions = async (
     5
   );
 
-  return createdUpdatedIssuePropertiesOptions;
+  return createdUpdatedIssuePropertiesOptions.filter((option) => option !== undefined) as ExIssuePropertyOption[];
 };

silo/src/etl/migrator/issue-types/issue-type.migrator.ts

@@ -16,7 +16,7 @@ type TCreateOrUpdateIssueTypes = {
 export const createOrUpdateIssueTypes = async (props: TCreateOrUpdateIssueTypes): Promise<ExIssueType[]> => {
   const { jobId, issueTypes, planeClient, workspaceSlug, projectId, method } = props;
 
-  const createOrUpdateIssueType = async (issueType: Partial<ExIssueType>) => {
+  const createOrUpdateIssueType = async (issueType: Partial<ExIssueType>): Promise<ExIssueType | undefined> => {
     try {
       let createdUpdatedIssueType: ExIssueType | undefined;
       if (method === "create") {
@@ -60,5 +60,5 @@ export const createOrUpdateIssueTypes = async (props: TCreateOrUpdateIssueTypes)
 
   const createdUpdatedIssueTypes = await processBatchPromises(issueTypes, createOrUpdateIssueType, 5);
 
-  return createdUpdatedIssueTypes?.filter((issueType) => issueType !== undefined) ?? [];
+  return createdUpdatedIssueTypes.filter((issueType) => issueType !== undefined) as ExIssueType[];
 };

silo/src/etl/migrator/labels.migrator.ts

@@ -11,7 +11,7 @@ export const createLabelsForIssues = async (
   workspaceSlug: string,
   projectId: string
 ): Promise<ExIssueLabel[]> => {
-  const createOrUpdateLabel = async (label: ExIssueLabel) => {
+  const createOrUpdateLabel = async (label: ExIssueLabel): Promise<ExIssueLabel | undefined> => {
     try {
       const createdLabel: ExIssueLabel | undefined = await protect(
         planeClient.label.create.bind(planeClient.label),
@@ -31,5 +31,5 @@ export const createLabelsForIssues = async (
   };
 
   const createdLabels = await processBatchPromises(labels, createOrUpdateLabel, 5);
-  return createdLabels?.filter((label) => label !== undefined) ?? [];
+  return createdLabels.filter((label) => label !== undefined) as ExIssueLabel[];
 };

silo/src/etl/migrator/modules.migrator.ts

@@ -91,7 +91,7 @@ export const createAllModules = async (
   workspaceSlug: string,
   projectId: string
 ): Promise<{ id: string; issues: string[] }[]> => {
-  const createOrUpdateModule = async (module: ExModule) => {
+  const createOrUpdateModule = async (module: ExModule): Promise<{ id: string; issues: string[] } | undefined> => {
     try {
       const createdModule = await protect(
         planeClient.modules.create.bind(planeClient.modules),
@@ -115,5 +115,5 @@ export const createAllModules = async (
 
   const createdModules = await processBatchPromises(modules, createOrUpdateModule, 5);
 
-  return createdModules?.filter((module) => module !== undefined) ?? [];
+  return createdModules.filter((module) => module !== undefined) as { id: string; issues: string[] }[];
 };

silo/src/etl/migrator/projects.migrator.ts

@@ -14,8 +14,8 @@ export const createProjects = async (
   planeClient: PlaneClient,
   workspaceSlug: string,
   existingProjects: ExProject[]
-) => {
-  const createProject = async (project: Partial<ExProject>) => {
+): Promise<ExProject[]> => {
+  const createProject = async (project: Partial<ExProject>): Promise<ExProject | undefined> => {
     try {
       const existingProject = existingProjects.find((exProject) => exProject.external_id === project.external_id);
       if (existingProject) {
@@ -45,7 +45,7 @@ export const createProjects = async (
 
   const createdProjects = await processBatchPromises(projects, createProject, 5);
 
-  return createdProjects?.filter((project) => project !== undefined) ?? [];
+  return createdProjects.filter((project) => project !== undefined) as ExProject[];
 };
 
 export const enableIssueTypeForProject = async (

silo/src/etl/migrator/states.migrator.ts

@@ -3,6 +3,11 @@ import { processBatchPromises } from "@/helpers/methods";
 import { protect } from "@/lib";
 import { logger } from "@/logger";
 
+type TCreateState = {
+  source_state: any;
+  target_state: Partial<ExState>;
+};
+
 /* ----------------------------- State Creation Utilities ----------------------------- */
 export const createStates = async (
   jobId: string,
@@ -12,7 +17,7 @@ export const createStates = async (
   projectId: string,
   existingStates: ExState[]
 ): Promise<{ source_state: any; target_state: Partial<ExState> }[]> => {
-  const createState = async (state: { source_state: any; target_state: Partial<ExState> }) => {
+  const createState = async (state: TCreateState): Promise<TCreateState | undefined> => {
     try {
       // check if the state already exists (by external_id or by name) BEFORE calling the API
       const existingStateById = existingStates.find(
@@ -51,5 +56,5 @@ export const createStates = async (
   // batch the state creation
   const createdStates = await processBatchPromises(states, createState, 2);
 
-  return createdStates?.filter((state) => state !== undefined) ?? [];
+  return createdStates.filter((state) => state !== undefined) as TCreateState[];
 };

silo/src/lib/errors.ts

@@ -80,6 +80,10 @@ export async function protect<T>(fn: (...args: any[]) => Promise<T>, ...args: an
         logger.info(`Rate limit exceeded (429). Retrying in ${Math.ceil(backoffTime / 1000)} seconds...`);
         await wait(backoffTime);
         continue;
+      } else if (error instanceof AxiosError && error.response?.status === 502) {
+        logger.info("502 error ====== in protect, retrying in 5 seconds...");
+        await wait(5000);
+        continue;
       }
 
       throw error;