Skip to content

Commit ba528df

Browse files
stephaniewang526stephaniewang526
authored
feat: add support for DecimalTargetTypes (#1345)
* feat: add support for DecimalTargetType Towards b/188414733 Implemented in LoadJobConfiguration * update test to verify that the destination table's decimal column is loaded as BIGNUMERIC type * add decimalTargetTypes support in ExternalTableDefinition.java * add decimalTargetTypes support in ExternalTableDefinition.java
1 parent 5f6307f commit ba528df

File tree

6 files changed

+114
-24
lines changed

6 files changed

+114
-24
lines changed

‎google-cloud-bigquery/clirr-ignored-differences.xml

Lines changed: 4 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -4,32 +4,12 @@
44
<!-- TODO: REMOVE AFTER RELEASE -->
55
<difference>
66
<differenceType>7013</differenceType>
7-
<className>com/google/cloud/bigquery/MaterializedViewDefinition</className>
8-
<method>com.google.cloud.bigquery.Clustering getClustering()</method>
7+
<className>com/google/cloud/bigquery/ExternalTableDefinition</className>
8+
<method>com.google.common.collect.ImmutableList getDecimalTargetTypes()</method>
99
</difference>
1010
<difference>
1111
<differenceType>7013</differenceType>
12-
<className>com/google/cloud/bigquery/MaterializedViewDefinition</className>
13-
<method>com.google.cloud.bigquery.RangePartitioning getRangePartitioning()</method>
14-
</difference>
15-
<difference>
16-
<differenceType>7013</differenceType>
17-
<className>com/google/cloud/bigquery/MaterializedViewDefinition</className>
18-
<method>com.google.cloud.bigquery.TimePartitioning getTimePartitioning()</method>
19-
</difference>
20-
<difference>
21-
<differenceType>7013</differenceType>
22-
<className>com/google/cloud/bigquery/MaterializedViewDefinition$Builder</className>
23-
<method>com.google.cloud.bigquery.MaterializedViewDefinition$Builder setClustering(com.google.cloud.bigquery.Clustering)</method>
24-
</difference>
25-
<difference>
26-
<differenceType>7013</differenceType>
27-
<className>com/google/cloud/bigquery/MaterializedViewDefinition$Builder</className>
28-
<method>com.google.cloud.bigquery.MaterializedViewDefinition$Builder setRangePartitioning(com.google.cloud.bigquery.RangePartitioning)</method>
29-
</difference>
30-
<difference>
31-
<differenceType>7013</differenceType>
32-
<className>com/google/cloud/bigquery/MaterializedViewDefinition$Builder</className>
33-
<method>com.google.cloud.bigquery.MaterializedViewDefinition$Builder setTimePartitioning(com.google.cloud.bigquery.TimePartitioning)</method>
12+
<className>com/google/cloud/bigquery/ExternalTableDefinition$Builder</className>
13+
<method> com.google.cloud.bigquery.ExternalTableDefinition$Builder setDecimalTargetTypes(java.util.List)</method>
3414
</difference>
3515
</differences>

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,17 @@ public Builder setFormatOptions(FormatOptions formatOptions) {
9393
return setFormatOptionsInner(formatOptions);
9494
}
9595

96+
/**
97+
* Defines the list of possible SQL data types to which the source decimal values are converted.
98+
* This list and the precision and the scale parameters of the decimal field determine the
99+
* target type. In the order of NUMERIC, BIGNUMERIC, and STRING, a type is picked if it is in
100+
* the specified list and if it supports the precision and the scale. STRING supports all
101+
* precision and scale values.
102+
*
103+
* @param decimalTargetTypes decimalTargetType or {@code null} for none
104+
*/
105+
public abstract Builder setDecimalTargetTypes(List<String> decimalTargetTypes);
106+
96107
abstract Builder setFormatOptionsInner(FormatOptions formatOptions);
97108

98109
/**
@@ -229,6 +240,9 @@ public <F extends FormatOptions> F getFormatOptions() {
229240
@Nullable
230241
abstract FormatOptions getFormatOptionsInner();
231242

243+
@Nullable
244+
public abstract ImmutableList<String> getDecimalTargetTypes();
245+
232246
/**
233247
* [Experimental] Returns whether automatic detection of schema and format options should be
234248
* performed.
@@ -283,6 +297,9 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC
283297
if (getSourceUris() != null) {
284298
externalConfigurationPb.setSourceUris(getSourceUris());
285299
}
300+
if (getDecimalTargetTypes() != null) {
301+
externalConfigurationPb.setDecimalTargetTypes(getDecimalTargetTypes());
302+
}
286303
if (getFormatOptions() != null && FormatOptions.CSV.equals(getFormatOptions().getType())) {
287304
externalConfigurationPb.setCsvOptions(((CsvOptions) getFormatOptions()).toPb());
288305
}
@@ -430,6 +447,10 @@ static ExternalTableDefinition fromPb(Table tablePb) {
430447
if (externalDataConfiguration.getSourceUris() != null) {
431448
builder.setSourceUris(ImmutableList.copyOf(externalDataConfiguration.getSourceUris()));
432449
}
450+
if (externalDataConfiguration.getDecimalTargetTypes() != null) {
451+
builder.setDecimalTargetTypes(
452+
ImmutableList.copyOf(externalDataConfiguration.getDecimalTargetTypes()));
453+
}
433454
if (externalDataConfiguration.getSourceFormat() != null) {
434455
builder.setFormatOptions(FormatOptions.of(externalDataConfiguration.getSourceFormat()));
435456
}
@@ -469,6 +490,9 @@ static ExternalTableDefinition fromExternalDataConfiguration(
469490
if (externalDataConfiguration.getSourceUris() != null) {
470491
builder.setSourceUris(externalDataConfiguration.getSourceUris());
471492
}
493+
if (externalDataConfiguration.getDecimalTargetTypes() != null) {
494+
builder.setDecimalTargetTypes(externalDataConfiguration.getDecimalTargetTypes());
495+
}
472496
if (externalDataConfiguration.getSchema() != null) {
473497
builder.setSchema(Schema.fromPb(externalDataConfiguration.getSchema()));
474498
}

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ public final class LoadJobConfiguration extends JobConfiguration implements Load
3838

3939
private final List<String> sourceUris;
4040
private final TableId destinationTable;
41+
private final List<String> decimalTargetTypes;
4142
private final EncryptionConfiguration destinationEncryptionConfiguration;
4243
private final JobInfo.CreateDisposition createDisposition;
4344
private final JobInfo.WriteDisposition writeDisposition;
@@ -61,6 +62,7 @@ public static final class Builder extends JobConfiguration.Builder<LoadJobConfig
6162

6263
private List<String> sourceUris;
6364
private TableId destinationTable;
65+
private List<String> decimalTargetTypes;
6466
private EncryptionConfiguration destinationEncryptionConfiguration;
6567
private JobInfo.CreateDisposition createDisposition;
6668
private JobInfo.WriteDisposition writeDisposition;
@@ -87,6 +89,7 @@ private Builder() {
8789
private Builder(LoadJobConfiguration loadConfiguration) {
8890
this();
8991
this.destinationTable = loadConfiguration.destinationTable;
92+
this.decimalTargetTypes = loadConfiguration.decimalTargetTypes;
9093
this.createDisposition = loadConfiguration.createDisposition;
9194
this.writeDisposition = loadConfiguration.writeDisposition;
9295
this.formatOptions = loadConfiguration.formatOptions;
@@ -112,6 +115,9 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur
112115
this();
113116
JobConfigurationLoad loadConfigurationPb = configurationPb.getLoad();
114117
this.destinationTable = TableId.fromPb(loadConfigurationPb.getDestinationTable());
118+
if (loadConfigurationPb.getDecimalTargetTypes() != null) {
119+
this.decimalTargetTypes = ImmutableList.copyOf(loadConfigurationPb.getDecimalTargetTypes());
120+
}
115121
if (loadConfigurationPb.getCreateDisposition() != null) {
116122
this.createDisposition =
117123
JobInfo.CreateDisposition.valueOf(loadConfigurationPb.getCreateDisposition());
@@ -278,6 +284,20 @@ public Builder setSourceUris(List<String> sourceUris) {
278284
return this;
279285
}
280286

287+
/**
288+
* Defines the list of possible SQL data types to which the source decimal values are converted.
289+
* This list and the precision and the scale parameters of the decimal field determine the
290+
* target type. In the order of NUMERIC, BIGNUMERIC, and STRING, a type is picked if it is in
291+
* the specified list and if it supports the precision and the scale. STRING supports all
292+
* precision and scale values.
293+
*
294+
* @param decimalTargetTypes decimalTargetType or {@code null} for none
295+
*/
296+
public Builder setDecimalTargetTypes(List<String> decimalTargetTypes) {
297+
this.decimalTargetTypes = decimalTargetTypes;
298+
return this;
299+
}
300+
281301
public Builder setAutodetect(Boolean autodetect) {
282302
this.autodetect = autodetect;
283303
return this;
@@ -341,6 +361,7 @@ private LoadJobConfiguration(Builder builder) {
341361
super(builder);
342362
this.sourceUris = builder.sourceUris;
343363
this.destinationTable = builder.destinationTable;
364+
this.decimalTargetTypes = builder.decimalTargetTypes;
344365
this.createDisposition = builder.createDisposition;
345366
this.writeDisposition = builder.writeDisposition;
346367
this.formatOptions = builder.formatOptions;
@@ -430,6 +451,10 @@ public List<String> getSourceUris() {
430451
return sourceUris;
431452
}
432453

454+
public List<String> getDecimalTargetTypes() {
455+
return decimalTargetTypes;
456+
}
457+
433458
public Boolean getAutodetect() {
434459
return autodetect;
435460
}
@@ -482,6 +507,7 @@ public Builder toBuilder() {
482507
ToStringHelper toStringHelper() {
483508
return super.toStringHelper()
484509
.add("destinationTable", destinationTable)
510+
.add("decimalTargetTypes", decimalTargetTypes)
485511
.add("destinationEncryptionConfiguration", destinationEncryptionConfiguration)
486512
.add("createDisposition", createDisposition)
487513
.add("writeDisposition", writeDisposition)
@@ -568,6 +594,9 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() {
568594
if (sourceUris != null) {
569595
loadConfigurationPb.setSourceUris(ImmutableList.copyOf(sourceUris));
570596
}
597+
if (decimalTargetTypes != null) {
598+
loadConfigurationPb.setDecimalTargetTypes(ImmutableList.copyOf(decimalTargetTypes));
599+
}
571600
if (schemaUpdateOptions != null) {
572601
ImmutableList.Builder<String> schemaUpdateOptionsBuilder = new ImmutableList.Builder<>();
573602
for (JobInfo.SchemaUpdateOption schemaUpdateOption : schemaUpdateOptions) {

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@
2727
public class ExternalTableDefinitionTest {
2828

2929
private static final List<String> SOURCE_URIS = ImmutableList.of("uri1", "uri2");
30+
private static final List<String> DECIMAL_TARGET_TYPES =
31+
ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING");
3032
private static final Field FIELD_SCHEMA1 =
3133
Field.newBuilder("StringField", LegacySQLTypeName.STRING)
3234
.setMode(Field.Mode.NULLABLE)
@@ -56,6 +58,7 @@ public class ExternalTableDefinitionTest {
5658
.build();
5759
private static final ExternalTableDefinition EXTERNAL_TABLE_DEFINITION =
5860
ExternalTableDefinition.newBuilder(SOURCE_URIS, TABLE_SCHEMA, CSV_OPTIONS)
61+
.setDecimalTargetTypes(DECIMAL_TARGET_TYPES)
5962
.setCompression(COMPRESSION)
6063
.setConnectionId(CONNECTION_ID)
6164
.setIgnoreUnknownValues(IGNORE_UNKNOWN_VALUES)
@@ -111,6 +114,7 @@ public void testBuilder() {
111114
assertEquals(MAX_BAD_RECORDS, EXTERNAL_TABLE_DEFINITION.getMaxBadRecords());
112115
assertEquals(TABLE_SCHEMA, EXTERNAL_TABLE_DEFINITION.getSchema());
113116
assertEquals(SOURCE_URIS, EXTERNAL_TABLE_DEFINITION.getSourceUris());
117+
assertEquals(DECIMAL_TARGET_TYPES, EXTERNAL_TABLE_DEFINITION.getDecimalTargetTypes());
114118
assertEquals(AUTODETECT, EXTERNAL_TABLE_DEFINITION.getAutodetect());
115119
assertEquals(HIVE_PARTITIONING_OPTIONS, EXTERNAL_TABLE_DEFINITION.getHivePartitioningOptions());
116120
assertNotEquals(EXTERNAL_TABLE_DEFINITION, TableDefinition.Type.EXTERNAL);
@@ -130,6 +134,7 @@ public void testToAndFromPb() {
130134
private void compareExternalTableDefinition(
131135
ExternalTableDefinition expected, ExternalTableDefinition value) {
132136
assertEquals(expected, value);
137+
assertEquals(expected.getDecimalTargetTypes(), value.getDecimalTargetTypes());
133138
assertEquals(expected.getCompression(), value.getCompression());
134139
assertEquals(expected.getConnectionId(), value.getConnectionId());
135140
assertEquals(expected.getFormatOptions(), value.getFormatOptions());

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ public class LoadJobConfigurationTest {
5050
.setDescription("FieldDescription")
5151
.build();
5252
private static final List<String> SOURCE_URIS = ImmutableList.of("uri1", "uri2");
53+
private static final List<String> DECIMAL_TARGET_TYPES =
54+
ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING");
5355
private static final List<SchemaUpdateOption> SCHEMA_UPDATE_OPTIONS =
5456
ImmutableList.of(SchemaUpdateOption.ALLOW_FIELD_ADDITION);
5557
private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA);
@@ -76,6 +78,7 @@ public class LoadJobConfigurationTest {
7678
.build();
7779
private static final LoadJobConfiguration LOAD_CONFIGURATION_CSV =
7880
LoadJobConfiguration.newBuilder(TABLE_ID, SOURCE_URIS)
81+
.setDecimalTargetTypes(DECIMAL_TARGET_TYPES)
7982
.setCreateDisposition(CREATE_DISPOSITION)
8083
.setWriteDisposition(WRITE_DISPOSITION)
8184
.setFormatOptions(CSV_OPTIONS)
@@ -228,6 +231,7 @@ private void compareLoadJobConfiguration(
228231
assertEquals(expected.hashCode(), value.hashCode());
229232
assertEquals(expected.toString(), value.toString());
230233
assertEquals(expected.getDestinationTable(), value.getDestinationTable());
234+
assertEquals(expected.getDecimalTargetTypes(), value.getDecimalTargetTypes());
231235
assertEquals(expected.getCreateDisposition(), value.getCreateDisposition());
232236
assertEquals(expected.getWriteDisposition(), value.getWriteDisposition());
233237
assertEquals(expected.getCsvOptions(), value.getCsvOptions());

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2747,6 +2747,54 @@ public void testLoadJobWithRangePartitioning() throws InterruptedException {
27472747
}
27482748
}
27492749

2750+
@Test
2751+
public void testLoadJobWithDecimalTargetTypes() throws InterruptedException {
2752+
String tableName = "test_load_job_table_parquet_decimalTargetTypes";
2753+
TableId destinationTable = TableId.of(DATASET, tableName);
2754+
String sourceUri = "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/numeric/numeric_38_12.parquet";
2755+
try {
2756+
LoadJobConfiguration configuration =
2757+
LoadJobConfiguration.newBuilder(destinationTable, sourceUri, FormatOptions.parquet())
2758+
.setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED)
2759+
.setDecimalTargetTypes(ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING"))
2760+
.build();
2761+
Job job = bigquery.create(JobInfo.of(configuration));
2762+
job = job.waitFor();
2763+
assertNull(job.getStatus().getError());
2764+
LoadJobConfiguration loadJobConfiguration = job.getConfiguration();
2765+
assertEquals(
2766+
ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING"),
2767+
loadJobConfiguration.getDecimalTargetTypes());
2768+
Table remoteTable = bigquery.getTable(DATASET, tableName);
2769+
assertNotNull(remoteTable);
2770+
assertEquals(
2771+
remoteTable.getDefinition().getSchema().getFields().get(0).getType().toString(),
2772+
"BIGNUMERIC");
2773+
} finally {
2774+
bigquery.delete(destinationTable);
2775+
}
2776+
}
2777+
2778+
@Test
2779+
public void testExternalTableWithDecimalTargetTypes() throws InterruptedException {
2780+
String tableName = "test_create_external_table_parquet_decimalTargetTypes";
2781+
TableId destinationTable = TableId.of(DATASET, tableName);
2782+
String sourceUri = "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/numeric/numeric_38_12.parquet";
2783+
ExternalTableDefinition externalTableDefinition =
2784+
ExternalTableDefinition.newBuilder(sourceUri, FormatOptions.parquet())
2785+
.setDecimalTargetTypes(ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING"))
2786+
.build();
2787+
TableInfo tableInfo = TableInfo.of(destinationTable, externalTableDefinition);
2788+
Table createdTable = bigquery.create(tableInfo);
2789+
assertNotNull(createdTable);
2790+
Table remoteTable = bigquery.getTable(DATASET, tableName);
2791+
assertNotNull(remoteTable);
2792+
assertEquals(
2793+
remoteTable.getDefinition().getSchema().getFields().get(0).getType().toString(),
2794+
"BIGNUMERIC");
2795+
assertTrue(remoteTable.delete());
2796+
}
2797+
27502798
@Test
27512799
public void testQueryJobWithDryRun() throws InterruptedException, TimeoutException {
27522800
String tableName = "test_query_job_table";

0 commit comments

Comments
 (0)