From eb0995f09dc646b42df978af7947079ef0d91ab8 Mon Sep 17 00:00:00 2001 From: Bobby Date: Mon, 3 May 2021 16:40:23 +1000 Subject: [PATCH] bigquery spark rate limit update --- ft_build/sparksLib/bigqueryIndex.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ft_build/sparksLib/bigqueryIndex.ts b/ft_build/sparksLib/bigqueryIndex.ts index 458767bc..b1c3f11b 100644 --- a/ft_build/sparksLib/bigqueryIndex.ts +++ b/ft_build/sparksLib/bigqueryIndex.ts @@ -344,12 +344,13 @@ const bigqueryIndex = async (payload, sparkContext) => { } catch (error) { if ( error?.errors?.length === 1 && - error?.errors?.[0]?.reason === "rateLimitExceeded" + (error?.errors?.[0]?.reason === "rateLimitExceeded" || + error?.errors?.[0]?.reason === "quotaExceeded") ) { const delay = Math.round( Math.floor(Math.random() * 3_000 * (delayDepth % 20) + 1000) ); - console.log(`API error: rateLimitExceeded, try again in ${delay}ms`); + console.log(`API rate limited, try again in ${delay}ms`); await sleep(delay); await executeQuery(query, delayDepth + 1); } else {