add derivative data type support for bigquery spark

ft_build/compiler/loader.ts

@@ -136,8 +136,13 @@ export const generateConfigFromTableSchema = async (
   // generate field types from table meta data
   const fieldTypes = JSON.stringify(
     Object.keys(schemaData.columns).reduce((acc, cur) => {
+      const field = schemaData.columns[cur];
+      let fieldType = field.type;
+      if (fieldType === "DERIVATIVE") {
+        fieldType = field.config.renderFieldType;
+      }
       return {
-        [cur]: schemaData.columns[cur].type,
+        [cur]: fieldType,
         ...acc,
       };
     }, {})

ft_build/sparksLib/bigqueryIndex.ts

@@ -130,7 +130,6 @@ const transformToSQLValue = (value: any, ftType: string) => {
     case "DOCUMENT_SELECT":
     case "SERVICE_SELECT":
     case "ACTION":
-    case "DERIVATIVE":
     case "AGGREGATE":
     default:
       // unknown or meaningless to sync