From 5e9f1352ddbe64fb00a89c29e6e008d92905dbe5 Mon Sep 17 00:00:00 2001 From: Saurabh Kumar <70131915+Saurabhkmr98@users.noreply.github.com> Date: Tue, 1 Jul 2025 14:45:09 +0530 Subject: [PATCH] [SILO-347] Add retry logic in case of 502 for plane apis #3547 --- .../issue-types/issue-property.migrator.ts | 16 ++++++++-------- .../migrator/issue-types/issue-type.migrator.ts | 4 ++-- silo/src/etl/migrator/labels.migrator.ts | 4 ++-- silo/src/etl/migrator/modules.migrator.ts | 4 ++-- silo/src/etl/migrator/projects.migrator.ts | 6 +++--- silo/src/etl/migrator/states.migrator.ts | 9 +++++++-- silo/src/lib/errors.ts | 4 ++++ 7 files changed, 28 insertions(+), 19 deletions(-) diff --git a/silo/src/etl/migrator/issue-types/issue-property.migrator.ts b/silo/src/etl/migrator/issue-types/issue-property.migrator.ts index cb84c376a9..191f0a4b90 100644 --- a/silo/src/etl/migrator/issue-types/issue-property.migrator.ts +++ b/silo/src/etl/migrator/issue-types/issue-property.migrator.ts @@ -31,14 +31,14 @@ export const createOrUpdateIssueProperties = async ( props; // Process a single issue property - const processIssueProperty = async (issueProperty: Partial): Promise => { + const processIssueProperty = async (issueProperty: Partial): Promise => { try { const issueType = issueTypesMap.get(issueProperty.type_id || "") || defaultIssueType; if (!issueType) { logger.error( `[${jobId.slice(0, 7)}] Issue type not found for the issue property: ${issueProperty.display_name}` ); - return null; + return undefined; } let createdUpdatedIssueProperty: ExIssueProperty | undefined; @@ -68,14 +68,14 @@ export const createOrUpdateIssueProperties = async ( `[${jobId.slice(0, 7)}] Error while ${method === "create" ? "creating" : "updating"} the issue property: ${issueProperty.display_name}`, error ); - return null; + return undefined; } }; // Process all issue properties in batches of 5 const createdUpdatedIssueProperties = await processBatchPromises(issueProperties, processIssueProperty, 5); - return createdUpdatedIssueProperties; + return createdUpdatedIssueProperties.filter((property) => property !== undefined) as ExIssueProperty[]; }; export const createOrUpdateIssuePropertiesOptions = async ( @@ -85,7 +85,7 @@ export const createOrUpdateIssuePropertiesOptions = async ( const processIssuePropertyOption = async ( issuePropertyOption: Partial - ): Promise => { + ): Promise => { let createdUpdatedIssuePropertyOption: ExIssuePropertyOption | undefined; try { const issueProperty = issuePropertyMap.get(issuePropertyOption.property_id || ""); @@ -93,7 +93,7 @@ export const createOrUpdateIssuePropertiesOptions = async ( logger.error( `[${jobId.slice(0, 7)}] Issue property not found for the issue property option: ${issuePropertyOption.name}` ); - return null; + return undefined; } if (method === "create") { @@ -121,7 +121,7 @@ export const createOrUpdateIssuePropertiesOptions = async ( `[${jobId.slice(0, 7)}] Error while ${method === "create" ? "creating" : "updating"} the issue property option: ${issuePropertyOption.name}`, error ); - return null; + return undefined; } }; @@ -131,5 +131,5 @@ export const createOrUpdateIssuePropertiesOptions = async ( 5 ); - return createdUpdatedIssuePropertiesOptions; + return createdUpdatedIssuePropertiesOptions.filter((option) => option !== undefined) as ExIssuePropertyOption[]; }; diff --git a/silo/src/etl/migrator/issue-types/issue-type.migrator.ts b/silo/src/etl/migrator/issue-types/issue-type.migrator.ts index fd8545c2f4..ab9745c104 100644 --- a/silo/src/etl/migrator/issue-types/issue-type.migrator.ts +++ b/silo/src/etl/migrator/issue-types/issue-type.migrator.ts @@ -16,7 +16,7 @@ type TCreateOrUpdateIssueTypes = { export const createOrUpdateIssueTypes = async (props: TCreateOrUpdateIssueTypes): Promise => { const { jobId, issueTypes, planeClient, workspaceSlug, projectId, method } = props; - const createOrUpdateIssueType = async (issueType: Partial) => { + const createOrUpdateIssueType = async (issueType: Partial): Promise => { try { let createdUpdatedIssueType: ExIssueType | undefined; if (method === "create") { @@ -60,5 +60,5 @@ export const createOrUpdateIssueTypes = async (props: TCreateOrUpdateIssueTypes) const createdUpdatedIssueTypes = await processBatchPromises(issueTypes, createOrUpdateIssueType, 5); - return createdUpdatedIssueTypes?.filter((issueType) => issueType !== undefined) ?? []; + return createdUpdatedIssueTypes.filter((issueType) => issueType !== undefined) as ExIssueType[]; }; diff --git a/silo/src/etl/migrator/labels.migrator.ts b/silo/src/etl/migrator/labels.migrator.ts index 80ee61500a..2e5aa8d149 100644 --- a/silo/src/etl/migrator/labels.migrator.ts +++ b/silo/src/etl/migrator/labels.migrator.ts @@ -11,7 +11,7 @@ export const createLabelsForIssues = async ( workspaceSlug: string, projectId: string ): Promise => { - const createOrUpdateLabel = async (label: ExIssueLabel) => { + const createOrUpdateLabel = async (label: ExIssueLabel): Promise => { try { const createdLabel: ExIssueLabel | undefined = await protect( planeClient.label.create.bind(planeClient.label), @@ -31,5 +31,5 @@ export const createLabelsForIssues = async ( }; const createdLabels = await processBatchPromises(labels, createOrUpdateLabel, 5); - return createdLabels?.filter((label) => label !== undefined) ?? []; + return createdLabels.filter((label) => label !== undefined) as ExIssueLabel[]; }; diff --git a/silo/src/etl/migrator/modules.migrator.ts b/silo/src/etl/migrator/modules.migrator.ts index 38b7a467b0..b3a3f317e4 100644 --- a/silo/src/etl/migrator/modules.migrator.ts +++ b/silo/src/etl/migrator/modules.migrator.ts @@ -91,7 +91,7 @@ export const createAllModules = async ( workspaceSlug: string, projectId: string ): Promise<{ id: string; issues: string[] }[]> => { - const createOrUpdateModule = async (module: ExModule) => { + const createOrUpdateModule = async (module: ExModule): Promise<{ id: string; issues: string[] } | undefined> => { try { const createdModule = await protect( planeClient.modules.create.bind(planeClient.modules), @@ -115,5 +115,5 @@ export const createAllModules = async ( const createdModules = await processBatchPromises(modules, createOrUpdateModule, 5); - return createdModules?.filter((module) => module !== undefined) ?? []; + return createdModules.filter((module) => module !== undefined) as { id: string; issues: string[] }[]; }; diff --git a/silo/src/etl/migrator/projects.migrator.ts b/silo/src/etl/migrator/projects.migrator.ts index 66b2a1ac1f..dd2d9f5766 100644 --- a/silo/src/etl/migrator/projects.migrator.ts +++ b/silo/src/etl/migrator/projects.migrator.ts @@ -14,8 +14,8 @@ export const createProjects = async ( planeClient: PlaneClient, workspaceSlug: string, existingProjects: ExProject[] -) => { - const createProject = async (project: Partial) => { +): Promise => { + const createProject = async (project: Partial): Promise => { try { const existingProject = existingProjects.find((exProject) => exProject.external_id === project.external_id); if (existingProject) { @@ -45,7 +45,7 @@ export const createProjects = async ( const createdProjects = await processBatchPromises(projects, createProject, 5); - return createdProjects?.filter((project) => project !== undefined) ?? []; + return createdProjects.filter((project) => project !== undefined) as ExProject[]; }; export const enableIssueTypeForProject = async ( diff --git a/silo/src/etl/migrator/states.migrator.ts b/silo/src/etl/migrator/states.migrator.ts index 8f466b7fc0..13bab8a20c 100644 --- a/silo/src/etl/migrator/states.migrator.ts +++ b/silo/src/etl/migrator/states.migrator.ts @@ -3,6 +3,11 @@ import { processBatchPromises } from "@/helpers/methods"; import { protect } from "@/lib"; import { logger } from "@/logger"; +type TCreateState = { + source_state: any; + target_state: Partial; +}; + /* ----------------------------- State Creation Utilities ----------------------------- */ export const createStates = async ( jobId: string, @@ -12,7 +17,7 @@ export const createStates = async ( projectId: string, existingStates: ExState[] ): Promise<{ source_state: any; target_state: Partial }[]> => { - const createState = async (state: { source_state: any; target_state: Partial }) => { + const createState = async (state: TCreateState): Promise => { try { // check if the state already exists (by external_id or by name) BEFORE calling the API const existingStateById = existingStates.find( @@ -51,5 +56,5 @@ export const createStates = async ( // batch the state creation const createdStates = await processBatchPromises(states, createState, 2); - return createdStates?.filter((state) => state !== undefined) ?? []; + return createdStates.filter((state) => state !== undefined) as TCreateState[]; }; diff --git a/silo/src/lib/errors.ts b/silo/src/lib/errors.ts index d02fa9b651..027d427632 100644 --- a/silo/src/lib/errors.ts +++ b/silo/src/lib/errors.ts @@ -80,6 +80,10 @@ export async function protect(fn: (...args: any[]) => Promise, ...args: an logger.info(`Rate limit exceeded (429). Retrying in ${Math.ceil(backoffTime / 1000)} seconds...`); await wait(backoffTime); continue; + } else if (error instanceof AxiosError && error.response?.status === 502) { + logger.info("502 error ====== in protect, retrying in 5 seconds..."); + await wait(5000); + continue; } throw error;