feat: support and optionally parse JSON field (#1229)

Author: alvarowolfx

samples/insertingDataTypes.js

@@ -45,6 +45,10 @@ function main(datasetId = 'my_dataset', tableId = 'my_table') {
         name: 'school',
         type: 'BYTES',
       },
+      {
+        name: 'metadata',
+        type: 'JSON',
+      },
       {
         name: 'location',
         type: 'GEOGRAPHY',
@@ -108,14 +112,19 @@ function main(datasetId = 'my_dataset', tableId = 'my_table') {
     const bqTimestamp = bigquery.timestamp('2020-04-27T18:07:25.356Z');
     const bqGeography = bigquery.geography('POINT(1 2)');
     const schoolBuffer = Buffer.from('Test University');
-
+    // a JSON field needs to be converted to a string
+    const metadata = JSON.stringify({
+      owner: 'John Doe',
+      contact: '[email protected]',
+    });
     // Rows to be inserted into table
     const rows = [
       {
         name: 'Tom',
         age: '30',
         location: bqGeography,
         school: schoolBuffer,
+        metadata: metadata,
         measurements: [50.05, 100.5],
         datesTimes: {
           day: bqDate,

src/bigquery.ts

@@ -104,6 +104,7 @@ export type Query = JobRequest<bigquery.IJobConfigurationQuery> & {
   jobTimeoutMs?: number;
   pageToken?: string;
   wrapIntegers?: boolean | IntegerTypeCastOptions;
+  parseJSON?: boolean;
 };
 
 export type QueryParamTypeStruct = {
@@ -122,6 +123,7 @@ export type QueryParamTypes =
 export type QueryOptions = QueryResultsOptions;
 export type QueryStreamOptions = {
   wrapIntegers?: boolean | IntegerTypeCastOptions;
+  parseJSON?: boolean;
 };
 export type DatasetResource = bigquery.IDataset;
 export type ValueType = bigquery.IQueryParameterType;
@@ -476,24 +478,29 @@ export class BigQuery extends Service {
    *
    * @param {object} schema
    * @param {array} rows
-   * @param {boolean|IntegerTypeCastOptions} wrapIntegers Wrap values of
+   * @param {object} options
+   * @param {boolean|IntegerTypeCastOptions} options.wrapIntegers Wrap values of
    *     'INT64' type in {@link BigQueryInt} objects.
    *     If a `boolean`, this will wrap values in {@link BigQueryInt} objects.
    *     If an `object`, this will return a value returned by
    *     `wrapIntegers.integerTypeCastFunction`.
    *     Please see {@link IntegerTypeCastOptions} for options descriptions.
-   * @param {array} selectedFields List of fields to return.
+   * @param {array} options.selectedFields List of fields to return.
    * If unspecified, all fields are returned.
+   * @param {array} options.parseJSON parse a 'JSON' field into a JSON object.
    * @returns Fields using their matching names from the table's schema.
    */
   static mergeSchemaWithRows_(
     schema: TableSchema | TableField,
     rows: TableRow[],
-    wrapIntegers: boolean | IntegerTypeCastOptions,
-    selectedFields?: string[]
+    options: {
+      wrapIntegers: boolean | IntegerTypeCastOptions;
+      selectedFields?: string[];
+      parseJSON?: boolean;
+    }
   ) {
-    if (selectedFields && selectedFields!.length > 0) {
-      const selectedFieldsArray = selectedFields!.map(c => {
+    if (options.selectedFields && options.selectedFields!.length > 0) {
+      const selectedFieldsArray = options.selectedFields!.map(c => {
         return c.split('.');
       });
 
@@ -505,7 +512,7 @@ export class BigQuery extends Service {
             .map(c => c!.toLowerCase())
             .indexOf(field.name!.toLowerCase()) >= 0
       );
-      selectedFields = selectedFieldsArray
+      options.selectedFields = selectedFieldsArray
         .filter(c => c.length > 0)
         .map(c => c.join('.'));
     }
@@ -518,10 +525,10 @@ export class BigQuery extends Service {
         let value = field.v;
         if (schemaField.mode === 'REPEATED') {
           value = (value as TableRowField[]).map(val => {
-            return convert(schemaField, val.v, wrapIntegers, selectedFields);
+            return convert(schemaField, val.v, options);
           });
         } else {
-          value = convert(schemaField, value, wrapIntegers, selectedFields);
+          value = convert(schemaField, value, options);
         }
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
         const fieldObject: any = {};
@@ -534,8 +541,11 @@ export class BigQuery extends Service {
       schemaField: TableField,
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
       value: any,
-      wrapIntegers: boolean | IntegerTypeCastOptions,
-      selectedFields?: string[]
+      options: {
+        wrapIntegers: boolean | IntegerTypeCastOptions;
+        selectedFields?: string[];
+        parseJSON?: boolean;
+      }
     ) {
       if (is.null(value)) {
         return value;
@@ -558,6 +568,7 @@ export class BigQuery extends Service {
         }
         case 'INTEGER':
         case 'INT64': {
+          const {wrapIntegers} = options;
           value = wrapIntegers
             ? typeof wrapIntegers === 'object'
               ? BigQuery.int(
@@ -580,8 +591,7 @@ export class BigQuery extends Service {
           value = BigQuery.mergeSchemaWithRows_(
             schemaField,
             value,
-            wrapIntegers,
-            selectedFields
+            options
           ).pop();
           break;
         }
@@ -605,6 +615,11 @@ export class BigQuery extends Service {
           value = BigQuery.geography(value);
           break;
         }
+        case 'JSON': {
+          const {parseJSON} = options;
+          value = parseJSON ? JSON.parse(value) : value;
+          break;
+        }
         default:
           break;
       }
@@ -1311,6 +1326,7 @@ export class BigQuery extends Service {
    *     the format of the {@link https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets#DatasetReference| `DatasetReference`}
    * @param {boolean} [options.wrapIntegers] Optionally wrap INT64 in BigQueryInt
    *     or custom INT64 value type.
+   * @param {boolean} [options.parseJSON] Optionally parse JSON as a JSON Object.
    * @param {object|array} [options.params] Option to provide query prarameters.
    * @param {JobCallback} [callback] The callback function.
    * @param {?error} callback.err An error returned while making this request.
@@ -2041,6 +2057,7 @@ export class BigQuery extends Service {
       typeof query === 'object'
         ? {
             wrapIntegers: query.wrapIntegers,
+            parseJSON: query.parseJSON,
           }
         : {};
     const callback =
@@ -2073,20 +2090,22 @@ export class BigQuery extends Service {
       return;
     }
 
-    const {location, maxResults, pageToken, wrapIntegers} = query;
+    const {location, maxResults, pageToken, wrapIntegers, parseJSON} = query;
 
     const opts = {
       location,
       maxResults,
       pageToken,
       wrapIntegers,
+      parseJSON,
       autoPaginate: false,
     };
 
     delete query.location;
     delete query.maxResults;
     delete query.pageToken;
     delete query.wrapIntegers;
+    delete query.parseJSON;
 
     this.query(query, opts, callback);
   }

src/job.ts

@@ -49,6 +49,7 @@ export type CancelResponse = [bigquery.IJobCancelResponse];
 export type QueryResultsOptions = {
   job?: Job;
   wrapIntegers?: boolean | IntegerTypeCastOptions;
+  parseJSON?: boolean;
 } & PagedRequest<bigquery.jobs.IGetQueryResultsParams>;
 
 /**
@@ -538,6 +539,8 @@ class Job extends Operation {
 
     const wrapIntegers = qs.wrapIntegers ? qs.wrapIntegers : false;
     delete qs.wrapIntegers;
+    const parseJSON = qs.parseJSON ? qs.parseJSON : false;
+    delete qs.parseJSON;
 
     delete qs.job;
 
@@ -559,11 +562,10 @@ class Job extends Operation {
         let rows: any = [];
 
         if (resp.schema && resp.rows) {
-          rows = BigQuery.mergeSchemaWithRows_(
-            resp.schema,
-            resp.rows,
-            wrapIntegers
-          );
+          rows = BigQuery.mergeSchemaWithRows_(resp.schema, resp.rows, {
+            wrapIntegers,
+            parseJSON,
+          });
         }
 
         let nextQuery: QueryResultsOptions | null = null;

src/table.ts

@@ -114,6 +114,7 @@ export type TableRowValue = string | TableRow;
 
 export type GetRowsOptions = PagedRequest<bigquery.tabledata.IListParams> & {
   wrapIntegers?: boolean | IntegerTypeCastOptions;
+  parseJSON?: boolean;
 };
 
 export type JobLoadMetadata = JobRequest<bigquery.IJobConfigurationLoad> & {
@@ -1811,6 +1812,8 @@ class Table extends ServiceObject {
       typeof optionsOrCallback === 'function' ? optionsOrCallback : cb;
     const wrapIntegers = options.wrapIntegers ? options.wrapIntegers : false;
     delete options.wrapIntegers;
+    const parseJSON = options.parseJSON ? options.parseJSON : false;
+    delete options.parseJSON;
     const onComplete = (
       err: Error | null,
       rows: TableRow[] | null,
@@ -1821,12 +1824,13 @@ class Table extends ServiceObject {
         callback!(err, null, null, resp);
         return;
       }
-      rows = BigQuery.mergeSchemaWithRows_(
-        this.metadata.schema,
-        rows || [],
-        wrapIntegers,
-        options.selectedFields ? options.selectedFields!.split(',') : []
-      );
+      rows = BigQuery.mergeSchemaWithRows_(this.metadata.schema, rows || [], {
+        wrapIntegers: wrapIntegers,
+        selectedFields: options.selectedFields
+          ? options.selectedFields!.split(',')
+          : [],
+        parseJSON,
+      });
       callback!(null, rows, nextQuery, resp);
     };