Skip to content

Commit 601c77a

Browse files
alvarowolfxalvarowolfx
authored
feat(bigquery): add default partition expiration to Dataset (#7096)
Resolves #7021
1 parent eddaf71 commit 601c77a

File tree

3 files changed

+93
-32
lines changed

3 files changed

+93
-32
lines changed

‎bigquery/dataset.go

+30-10
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,10 @@ type DatasetMetadata struct {
4545
Access []*AccessEntry // Access permissions.
4646
DefaultEncryptionConfig *EncryptionConfig
4747

48+
// DefaultPartitionExpiration is the default expiration time for
49+
// all newly created partitioned tables in the dataset.
50+
DefaultPartitionExpiration time.Duration
51+
4852
// These fields are read-only.
4953
CreationTime time.Time
5054
LastModifiedTime time.Time // When the dataset or any of its tables were modified.
@@ -91,6 +95,11 @@ type DatasetMetadataToUpdate struct {
9195
// If set to time.Duration(0), new tables never expire.
9296
DefaultTableExpiration optional.Duration
9397

98+
// DefaultTableExpiration is the default expiration time for
99+
// all newly created partitioned tables.
100+
// If set to time.Duration(0), new table partitions never expire.
101+
DefaultPartitionExpiration optional.Duration
102+
94103
// DefaultEncryptionConfig defines CMEK settings for new resources created
95104
// in the dataset.
96105
DefaultEncryptionConfig *EncryptionConfig
@@ -164,6 +173,7 @@ func (dm *DatasetMetadata) toBQ() (*bq.Dataset, error) {
164173
ds.Description = dm.Description
165174
ds.Location = dm.Location
166175
ds.DefaultTableExpirationMs = int64(dm.DefaultTableExpiration / time.Millisecond)
176+
ds.DefaultPartitionExpirationMs = int64(dm.DefaultPartitionExpiration / time.Millisecond)
167177
ds.Labels = dm.Labels
168178
var err error
169179
ds.Access, err = accessListToBQ(dm.Access)
@@ -245,16 +255,17 @@ func (d *Dataset) Metadata(ctx context.Context) (md *DatasetMetadata, err error)
245255

246256
func bqToDatasetMetadata(d *bq.Dataset, c *Client) (*DatasetMetadata, error) {
247257
dm := &DatasetMetadata{
248-
CreationTime: unixMillisToTime(d.CreationTime),
249-
LastModifiedTime: unixMillisToTime(d.LastModifiedTime),
250-
DefaultTableExpiration: time.Duration(d.DefaultTableExpirationMs) * time.Millisecond,
251-
DefaultEncryptionConfig: bqToEncryptionConfig(d.DefaultEncryptionConfiguration),
252-
Description: d.Description,
253-
Name: d.FriendlyName,
254-
FullID: d.Id,
255-
Location: d.Location,
256-
Labels: d.Labels,
257-
ETag: d.Etag,
258+
CreationTime: unixMillisToTime(d.CreationTime),
259+
LastModifiedTime: unixMillisToTime(d.LastModifiedTime),
260+
DefaultTableExpiration: time.Duration(d.DefaultTableExpirationMs) * time.Millisecond,
261+
DefaultPartitionExpiration: time.Duration(d.DefaultPartitionExpirationMs) * time.Millisecond,
262+
DefaultEncryptionConfig: bqToEncryptionConfig(d.DefaultEncryptionConfiguration),
263+
Description: d.Description,
264+
Name: d.FriendlyName,
265+
FullID: d.Id,
266+
Location: d.Location,
267+
Labels: d.Labels,
268+
ETag: d.Etag,
258269
}
259270
for _, a := range d.Access {
260271
e, err := bqToAccessEntry(a, c)
@@ -324,6 +335,15 @@ func (dm *DatasetMetadataToUpdate) toBQ() (*bq.Dataset, error) {
324335
ds.DefaultTableExpirationMs = int64(dur / time.Millisecond)
325336
}
326337
}
338+
if dm.DefaultPartitionExpiration != nil {
339+
dur := optional.ToDuration(dm.DefaultPartitionExpiration)
340+
if dur == 0 {
341+
// Send a null to delete the field.
342+
ds.NullFields = append(ds.NullFields, "DefaultPartitionExpirationMs")
343+
} else {
344+
ds.DefaultPartitionExpirationMs = int64(dur / time.Millisecond)
345+
}
346+
}
327347
if dm.DefaultEncryptionConfig != nil {
328348
ds.DefaultEncryptionConfiguration = dm.DefaultEncryptionConfig.toBQ()
329349
ds.DefaultEncryptionConfiguration.ForceSendFields = []string{"KmsKeyName"}

‎bigquery/dataset_integration_test.go

+35
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,41 @@ func TestIntegration_DatasetUpdateDefaultExpiration(t *testing.T) {
194194
}
195195
}
196196

197+
func TestIntegration_DatasetUpdateDefaultPartitionExpiration(t *testing.T) {
198+
if client == nil {
199+
t.Skip("Integration tests skipped")
200+
}
201+
ctx := context.Background()
202+
_, err := dataset.Metadata(ctx)
203+
if err != nil {
204+
t.Fatal(err)
205+
}
206+
// Set the default partition expiration time.
207+
md, err := dataset.Update(ctx, DatasetMetadataToUpdate{DefaultPartitionExpiration: 24 * time.Hour}, "")
208+
if err != nil {
209+
t.Fatal(err)
210+
}
211+
if md.DefaultPartitionExpiration != 24*time.Hour {
212+
t.Fatalf("got %v, want 24h", md.DefaultPartitionExpiration)
213+
}
214+
// Omitting DefaultPartitionExpiration doesn't change it.
215+
md, err = dataset.Update(ctx, DatasetMetadataToUpdate{Name: "xyz"}, "")
216+
if err != nil {
217+
t.Fatal(err)
218+
}
219+
if md.DefaultPartitionExpiration != 24*time.Hour {
220+
t.Fatalf("got %s, want 24h", md.DefaultPartitionExpiration)
221+
}
222+
// Setting it to 0 deletes it (which looks like a 0 duration).
223+
md, err = dataset.Update(ctx, DatasetMetadataToUpdate{DefaultPartitionExpiration: time.Duration(0)}, "")
224+
if err != nil {
225+
t.Fatal(err)
226+
}
227+
if md.DefaultPartitionExpiration != 0 {
228+
t.Fatalf("got %s, want 0", md.DefaultPartitionExpiration)
229+
}
230+
}
231+
197232
func TestIntegration_DatasetUpdateAccess(t *testing.T) {
198233
if client == nil {
199234
t.Skip("Integration tests skipped")

‎bigquery/dataset_test.go

+28-22
Original file line numberDiff line numberDiff line change
@@ -319,9 +319,10 @@ func TestDatasetToBQ(t *testing.T) {
319319
{nil, &bq.Dataset{}},
320320
{&DatasetMetadata{Name: "name"}, &bq.Dataset{FriendlyName: "name"}},
321321
{&DatasetMetadata{
322-
Name: "name",
323-
Description: "desc",
324-
DefaultTableExpiration: time.Hour,
322+
Name: "name",
323+
Description: "desc",
324+
DefaultTableExpiration: time.Hour,
325+
DefaultPartitionExpiration: 24 * time.Hour,
325326
DefaultEncryptionConfig: &EncryptionConfig{
326327
KMSKeyName: "some_key",
327328
},
@@ -338,9 +339,10 @@ func TestDatasetToBQ(t *testing.T) {
338339
},
339340
},
340341
}, &bq.Dataset{
341-
FriendlyName: "name",
342-
Description: "desc",
343-
DefaultTableExpirationMs: 60 * 60 * 1000,
342+
FriendlyName: "name",
343+
Description: "desc",
344+
DefaultTableExpirationMs: 60 * 60 * 1000,
345+
DefaultPartitionExpirationMs: 24 * 60 * 60 * 1000,
344346
DefaultEncryptionConfiguration: &bq.EncryptionConfiguration{
345347
KmsKeyName: "some_key",
346348
},
@@ -390,11 +392,12 @@ func TestBQToDatasetMetadata(t *testing.T) {
390392
mTime := time.Date(2017, 10, 31, 0, 0, 0, 0, time.Local)
391393
mMillis := mTime.UnixNano() / 1e6
392394
q := &bq.Dataset{
393-
CreationTime: cMillis,
394-
LastModifiedTime: mMillis,
395-
FriendlyName: "name",
396-
Description: "desc",
397-
DefaultTableExpirationMs: 60 * 60 * 1000,
395+
CreationTime: cMillis,
396+
LastModifiedTime: mMillis,
397+
FriendlyName: "name",
398+
Description: "desc",
399+
DefaultTableExpirationMs: 60 * 60 * 1000,
400+
DefaultPartitionExpirationMs: 24 * 60 * 60 * 1000,
398401
DefaultEncryptionConfiguration: &bq.EncryptionConfiguration{
399402
KmsKeyName: "some_key",
400403
},
@@ -420,11 +423,12 @@ func TestBQToDatasetMetadata(t *testing.T) {
420423
Etag: "etag",
421424
}
422425
want := &DatasetMetadata{
423-
CreationTime: cTime,
424-
LastModifiedTime: mTime,
425-
Name: "name",
426-
Description: "desc",
427-
DefaultTableExpiration: time.Hour,
426+
CreationTime: cTime,
427+
LastModifiedTime: mTime,
428+
Name: "name",
429+
Description: "desc",
430+
DefaultTableExpiration: time.Hour,
431+
DefaultPartitionExpiration: 24 * time.Hour,
428432
DefaultEncryptionConfig: &EncryptionConfig{
429433
KMSKeyName: "some_key",
430434
},
@@ -458,9 +462,10 @@ func TestBQToDatasetMetadata(t *testing.T) {
458462

459463
func TestDatasetMetadataToUpdateToBQ(t *testing.T) {
460464
dm := DatasetMetadataToUpdate{
461-
Description: "desc",
462-
Name: "name",
463-
DefaultTableExpiration: time.Hour,
465+
Description: "desc",
466+
Name: "name",
467+
DefaultTableExpiration: time.Hour,
468+
DefaultPartitionExpiration: 24 * time.Hour,
464469
DefaultEncryptionConfig: &EncryptionConfig{
465470
KMSKeyName: "some_key",
466471
},
@@ -473,9 +478,10 @@ func TestDatasetMetadataToUpdateToBQ(t *testing.T) {
473478
t.Fatal(err)
474479
}
475480
want := &bq.Dataset{
476-
Description: "desc",
477-
FriendlyName: "name",
478-
DefaultTableExpirationMs: 60 * 60 * 1000,
481+
Description: "desc",
482+
FriendlyName: "name",
483+
DefaultTableExpirationMs: 60 * 60 * 1000,
484+
DefaultPartitionExpirationMs: 24 * 60 * 60 * 1000,
479485
DefaultEncryptionConfiguration: &bq.EncryptionConfiguration{
480486
KmsKeyName: "some_key",
481487
ForceSendFields: []string{"KmsKeyName"},

0 commit comments

Comments
 (0)