Mirrors/rowy
1
0
Fork 0
mirror of https://github.com/rowyio/rowy.git synced 2026-02-24 04:01:17 +01:00
Code Issues Packages Projects Releases Wiki Activity

configurable projectID for bigquery

Browse Source
This commit is contained in:
Shams mosowi
2021-06-22 22:45:21 +10:00
parent af6abfb92e
commit 4a22810009
1 changed files with 4 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
ft_build/sparksLib/bigqueryIndex.ts
View File

@@ -163,18 +163,17 @@ const transformToSQLType = (ftType: string) => {
};
const bigqueryIndex = async (payload, sparkContext) => {
const { row, objectID, index, fieldsToSync } = payload;
const { row, objectID, index, fieldsToSync, projectID } = payload;
const { triggerType, change, fieldTypes } = sparkContext;
const record = rowReducer(fieldsToSync, row);
const { BigQuery } = require("@google-cloud/bigquery");
const { getSecret } = require("../utils");
const bigquery = new BigQuery();
const { projectID } = await getSecret("bigquery");
const tableFullName = `${projectID}.firetable.${index}`;
const _projectID = projectID ?? process.env.GCLOUD_PROJECT;
const tableFullName = `${_projectID}.firetable.${index}`;
console.log(
`projectID: ${projectID}, index: ${index}, tableFullName: ${tableFullName}`
`projectID: ${_projectID}, index: ${index}, tableFullName: ${tableFullName}`
);
// create dataset with exact name "firetable" if not exists