feat: Add decimal target type (#2166)

* Add getDecimalTargetTypes and getDecimalTargetTypes

* Add getDecimalTargetTypes and getDecimalTargetTypes

* Modified unit test cases for DecimalTargetTypes

* Adding getDecimalTargetTypes and getDecimalTargetTypes as clirr exceptions. Removed previous exceptions

* Adding testInsertWithDecimalTargetTypes IT

Author: prash-mi

google-cloud-bigquery/clirr-ignored-differences.xml

@@ -2,34 +2,14 @@
 <!-- see http://www.mojohaus.org/clirr-maven-plugin/examples/ignored-differences.html -->
 <differences>
   <!-- TODO: REMOVE AFTER RELEASE -->
-  <difference>
-    <differenceType>7012</differenceType>
-    <className>com/google/cloud/bigquery/BigQuery</className>
-    <method>com.google.cloud.bigquery.Connection createConnection(com.google.cloud.bigquery.ConnectionSettings)</method>
-  </difference>
-  <difference>
-    <differenceType>7012</differenceType>
-    <className>com/google/cloud/bigquery/BigQuery</className>
-    <method>com.google.cloud.bigquery.Connection createConnection()</method>
-  </difference>
-  <difference>
-    <differenceType>7012</differenceType>
-    <className>com/google/cloud/bigquery/spi/v2/BigQueryRpc</className>
-    <method>com.google.api.services.bigquery.model.Job createJobForQuery(com.google.api.services.bigquery.model.Job)</method>
-  </difference>
-  <difference>
-    <differenceType>7012</differenceType>
-    <className>com/google/cloud/bigquery/spi/v2/BigQueryRpc</className>
-    <method>com.google.api.services.bigquery.model.Job getQueryJob(java.lang.String, java.lang.String, java.lang.String)</method>
-  </difference>
-  <difference>
+<difference>
     <differenceType>7012</differenceType>
-    <className>com/google/cloud/bigquery/spi/v2/BigQueryRpc</className>
-    <method>com.google.api.services.bigquery.model.GetQueryResultsResponse getQueryResultsWithRowLimit(java.lang.String, java.lang.String, java.lang.String, java.lang.Integer)</method>
+    <className>com/google/cloud/bigquery/LoadConfiguration</className>
+    <method>java.util.List getDecimalTargetTypes()</method>
   </difference>
   <difference>
     <differenceType>7012</differenceType>
-    <className>com/google/cloud/bigquery/spi/v2/BigQueryRpc</className>
-    <method>com.google.api.services.bigquery.model.TableDataList listTableDataWithRowLimit(java.lang.String, java.lang.String, java.lang.String, java.lang.Integer, java.lang.String)</method>
+    <className>com/google/cloud/bigquery/LoadConfiguration$Builder</className>
+    <method>com.google.cloud.bigquery.LoadConfiguration$Builder setDecimalTargetTypes(java.util.List)</method>
   </difference>
 </differences>

google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java

@@ -125,6 +125,17 @@ interface Builder {
      */
     Builder setUseAvroLogicalTypes(Boolean useAvroLogicalTypes);
 
+    /**
+     * Defines the list of possible SQL data types to which the source decimal values are converted.
+     * This list and the precision and the scale parameters of the decimal field determine the
+     * target type. In the order of NUMERIC, BIGNUMERIC, and STRING, a type is picked if it is in
+     * the specified list and if it supports the precision and the scale. STRING supports all
+     * precision and scale values.
+     *
+     * @param decimalTargetTypes decimalTargetType or {@code null} for none
+     */
+    Builder setDecimalTargetTypes(List<String> decimalTargetTypes);
+
     LoadConfiguration build();
   }
 
@@ -214,6 +225,15 @@ interface Builder {
   /** Returns True/False. Indicates whether the logical type is interpreted. */
   Boolean getUseAvroLogicalTypes();
 
+  /**
+   * Returns the list of possible SQL data types to which the source decimal values are converted.
+   * This list and the precision and the scale parameters of the decimal field determine the target
+   * type. In the order of NUMERIC, BIGNUMERIC, and STRING, a type is picked if it is in the
+   * specified list and if it supports the precision and the scale. STRING supports all precision
+   * and scale values.
+   */
+  List<String> getDecimalTargetTypes();
+
   /** Returns a builder for the load configuration object. */
   Builder toBuilder();
 }

google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java

@@ -55,6 +55,7 @@ public final class WriteChannelConfiguration implements LoadConfiguration, Seria
   private final Clustering clustering;
   private final Boolean useAvroLogicalTypes;
   private final Map<String, String> labels;
+  private List<String> decimalTargetTypes;
 
   public static final class Builder implements LoadConfiguration.Builder {
 
@@ -73,6 +74,7 @@ public static final class Builder implements LoadConfiguration.Builder {
     private Clustering clustering;
     private Boolean useAvroLogicalTypes;
     private Map<String, String> labels;
+    private List<String> decimalTargetTypes;
 
     private Builder() {}
 
@@ -93,6 +95,7 @@ private Builder(WriteChannelConfiguration writeChannelConfiguration) {
       this.clustering = writeChannelConfiguration.clustering;
       this.useAvroLogicalTypes = writeChannelConfiguration.useAvroLogicalTypes;
       this.labels = writeChannelConfiguration.labels;
+      this.decimalTargetTypes = writeChannelConfiguration.decimalTargetTypes;
     }
 
     private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) {
@@ -169,6 +172,9 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur
       if (configurationPb.getLabels() != null) {
         this.labels = configurationPb.getLabels();
       }
+      if (loadConfigurationPb.getDecimalTargetTypes() != null) {
+        this.decimalTargetTypes = loadConfigurationPb.getDecimalTargetTypes();
+      }
     }
 
     @Override
@@ -262,6 +268,12 @@ public Builder setLabels(Map<String, String> labels) {
       return this;
     }
 
+    @Override
+    public Builder setDecimalTargetTypes(List<String> decimalTargetTypes) {
+      this.decimalTargetTypes = decimalTargetTypes;
+      return this;
+    }
+
     @Override
     public WriteChannelConfiguration build() {
       return new WriteChannelConfiguration(this);
@@ -284,6 +296,7 @@ protected WriteChannelConfiguration(Builder builder) {
     this.clustering = builder.clustering;
     this.useAvroLogicalTypes = builder.useAvroLogicalTypes;
     this.labels = builder.labels;
+    this.decimalTargetTypes = builder.decimalTargetTypes;
   }
 
   @Override
@@ -372,6 +385,11 @@ public Map<String, String> getLabels() {
     return labels;
   }
 
+  @Override
+  public List<String> getDecimalTargetTypes() {
+    return decimalTargetTypes;
+  }
+
   @Override
   public Builder toBuilder() {
     return new Builder(this);
@@ -393,7 +411,8 @@ MoreObjects.ToStringHelper toStringHelper() {
         .add("timePartitioning", timePartitioning)
         .add("clustering", clustering)
         .add("useAvroLogicalTypes", useAvroLogicalTypes)
-        .add("labels", labels);
+        .add("labels", labels)
+        .add("decimalTargetTypes", decimalTargetTypes);
   }
 
   @Override
@@ -424,7 +443,8 @@ public int hashCode() {
         timePartitioning,
         clustering,
         useAvroLogicalTypes,
-        labels);
+        labels,
+        decimalTargetTypes);
   }
 
   WriteChannelConfiguration setProjectId(String projectId) {
@@ -495,6 +515,9 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() {
     if (labels != null) {
       jobConfiguration.setLabels(labels);
     }
+    if (decimalTargetTypes != null) {
+      loadConfigurationPb.setDecimalTargetTypes(decimalTargetTypes);
+    }
     jobConfiguration.setLoad(loadConfigurationPb);
     return jobConfiguration;
   }

google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/WriteChannelConfigurationTest.java

@@ -59,6 +59,8 @@ public class WriteChannelConfigurationTest {
       Clustering.newBuilder().setFields(ImmutableList.of("Foo", "Bar")).build();
   private static final Map<String, String> LABELS =
       ImmutableMap.of("test-job-name", "test-write-channel");
+  private static final List<String> DECIMAL_TARGET_TYPES =
+      ImmutableList.of("NUMERIC", "BIGNUMERIC");
   private static final WriteChannelConfiguration LOAD_CONFIGURATION_CSV =
       WriteChannelConfiguration.newBuilder(TABLE_ID)
           .setCreateDisposition(CREATE_DISPOSITION)
@@ -73,6 +75,7 @@ public class WriteChannelConfigurationTest {
           .setTimePartitioning(TIME_PARTITIONING)
           .setClustering(CLUSTERING)
           .setLabels(LABELS)
+          .setDecimalTargetTypes(DECIMAL_TARGET_TYPES)
           .build();
 
   private static final DatastoreBackupOptions BACKUP_OPTIONS =
@@ -104,6 +107,7 @@ public class WriteChannelConfigurationTest {
           .setTimePartitioning(TIME_PARTITIONING)
           .setClustering(CLUSTERING)
           .setUseAvroLogicalTypes(USERAVROLOGICALTYPES)
+          .setDecimalTargetTypes(DECIMAL_TARGET_TYPES)
           .build();
 
   @Test
@@ -169,7 +173,8 @@ public void testBuilder() {
             .setMaxBadRecords(MAX_BAD_RECORDS)
             .setSchemaUpdateOptions(SCHEMA_UPDATE_OPTIONS)
             .setSchema(TABLE_SCHEMA)
-            .setAutodetect(AUTODETECT);
+            .setAutodetect(AUTODETECT)
+            .setDecimalTargetTypes(DECIMAL_TARGET_TYPES);
     WriteChannelConfiguration loadConfigurationCSV = builder.build();
     assertEquals(TABLE_ID, loadConfigurationCSV.getDestinationTable());
     assertEquals(CREATE_DISPOSITION, loadConfigurationCSV.getCreateDisposition());
@@ -182,6 +187,7 @@ public void testBuilder() {
     assertEquals(TABLE_SCHEMA, loadConfigurationCSV.getSchema());
     assertEquals(SCHEMA_UPDATE_OPTIONS, loadConfigurationCSV.getSchemaUpdateOptions());
     assertEquals(AUTODETECT, loadConfigurationCSV.getAutodetect());
+    assertEquals(DECIMAL_TARGET_TYPES, loadConfigurationCSV.getDecimalTargetTypes());
     builder.setFormatOptions(BACKUP_OPTIONS);
     WriteChannelConfiguration loadConfigurationBackup = builder.build();
     assertEquals(BACKUP_OPTIONS, loadConfigurationBackup.getDatastoreBackupOptions());
@@ -225,5 +231,6 @@ private void compareLoadConfiguration(
     assertEquals(expected.getClustering(), value.getClustering());
     assertEquals(expected.getUseAvroLogicalTypes(), value.getUseAvroLogicalTypes());
     assertEquals(expected.getLabels(), value.getLabels());
+    assertEquals(expected.getDecimalTargetTypes(), value.getDecimalTargetTypes());
   }
 }

google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java

@@ -4374,6 +4374,32 @@ public void testInsertFromFileWithLabels()
     assertTrue(bigquery.delete(tableId));
   }
 
+  @Test
+  public void testInsertWithDecimalTargetTypes()
+      throws InterruptedException, IOException, TimeoutException {
+    String destinationTableName = "test_insert_from_file_table_with_decimal_target_type";
+    TableId tableId = TableId.of(DATASET, destinationTableName);
+    WriteChannelConfiguration configuration =
+        WriteChannelConfiguration.newBuilder(tableId)
+            .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED)
+            .setAutodetect(true)
+            .setDecimalTargetTypes(ImmutableList.of("STRING", "NUMERIC", "BIGNUMERIC"))
+            .build();
+    TableDataWriteChannel channel = bigquery.writer(configuration);
+    try {
+      channel.write(ByteBuffer.wrap("foo".getBytes(StandardCharsets.UTF_8)));
+    } finally {
+      channel.close();
+    }
+    Job job = channel.getJob().waitFor();
+    LoadJobConfiguration jobConfiguration = job.getConfiguration();
+    assertNull(job.getStatus().getError());
+    assertEquals(
+        ImmutableList.of("STRING", "NUMERIC", "BIGNUMERIC"),
+        jobConfiguration.getDecimalTargetTypes());
+    assertTrue(bigquery.delete(tableId));
+  }
+
   @Test
   public void testLocation() throws Exception {
     String location = "EU";