Skip to content

Commit 2821902

Browse files
ad548kolea2
andauthored
feat: add CMEK Support (#656)
* feat: Implement hand written layer on top of CMEK protos * fix tests * fix tests * fix: fix config file * fix: fix test * fix: fix and cleanup tests * fix: lint fix Co-authored-by: Kristen O'Leary <[email protected]>
1 parent 255cac6 commit 2821902

21 files changed

+984
-9
lines changed

.kokoro/nightly/integration.cfg

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ env_vars: {
88

99
env_vars: {
1010
key: "INTEGRATION_TEST_ARGS"
11-
value: "-P bigtable-emulator-it,bigtable-prod-it -Dbigtable.project=gcloud-devel -Dbigtable.instance=google-cloud-bigtable -Dbigtable.table=integration-tests"
11+
value: "-P bigtable-emulator-it,bigtable-prod-it -Dbigtable.project=gcloud-devel -Dbigtable.instance=google-cloud-bigtable -Dbigtable.table=integration-tests -Dbigtable.kms_key_name=projects/gcloud-devel/locations/us-central1/keyRings/cmek-test-key-ring/cryptoKeys/cmek-test-key -Dbigtable.wait-for-cmek-key-status=true"
1212
}
1313

1414
env_vars: {

.kokoro/presubmit/integration.cfg

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ env_vars: {
88

99
env_vars: {
1010
key: "INTEGRATION_TEST_ARGS"
11-
value: "-P bigtable-emulator-it,bigtable-prod-it -Dbigtable.project=gcloud-devel -Dbigtable.instance=google-cloud-bigtable -Dbigtable.table=integration-tests"
11+
value: "-P bigtable-emulator-it,bigtable-prod-it -Dbigtable.project=gcloud-devel -Dbigtable.instance=google-cloud-bigtable -Dbigtable.table=integration-tests -Dbigtable.kms_key_name=projects/gcloud-devel/locations/us-central1/keyRings/cmek-test-key-ring/cryptoKeys/cmek-test-key -Dbigtable.wait-for-cmek-key-status=true"
1212
}
1313

1414
env_vars: {

google-cloud-bigtable/pom.xml

+5
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,11 @@
214214
<artifactId>truth</artifactId>
215215
<scope>test</scope>
216216
</dependency>
217+
<dependency>
218+
<groupId>com.google.truth.extensions</groupId>
219+
<artifactId>truth-proto-extension</artifactId>
220+
<scope>test</scope>
221+
</dependency>
217222
<dependency>
218223
<groupId>io.grpc</groupId>
219224
<artifactId>grpc-testing</artifactId>

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClient.java

+52
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@
3030
import com.google.bigtable.admin.v2.ListBackupsRequest;
3131
import com.google.bigtable.admin.v2.ListTablesRequest;
3232
import com.google.bigtable.admin.v2.RestoreTableMetadata;
33+
import com.google.bigtable.admin.v2.Table.ClusterState;
34+
import com.google.bigtable.admin.v2.Table.View;
3335
import com.google.cloud.Policy;
3436
import com.google.cloud.Policy.DefaultMarshaller;
3537
import com.google.cloud.bigtable.admin.v2.BaseBigtableTableAdminClient.ListBackupsPage;
@@ -40,6 +42,7 @@
4042
import com.google.cloud.bigtable.admin.v2.models.Backup;
4143
import com.google.cloud.bigtable.admin.v2.models.CreateBackupRequest;
4244
import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest;
45+
import com.google.cloud.bigtable.admin.v2.models.EncryptionInfo;
4346
import com.google.cloud.bigtable.admin.v2.models.GCRules;
4447
import com.google.cloud.bigtable.admin.v2.models.ModifyColumnFamiliesRequest;
4548
import com.google.cloud.bigtable.admin.v2.models.OptimizeRestoredTableOperationToken;
@@ -49,6 +52,8 @@
4952
import com.google.cloud.bigtable.admin.v2.models.UpdateBackupRequest;
5053
import com.google.cloud.bigtable.admin.v2.stub.EnhancedBigtableTableAdminStub;
5154
import com.google.common.base.Preconditions;
55+
import com.google.common.collect.ImmutableList;
56+
import com.google.common.collect.ImmutableMap;
5257
import com.google.common.collect.Lists;
5358
import com.google.common.util.concurrent.MoreExecutors;
5459
import com.google.iam.v1.GetIamPolicyRequest;
@@ -60,6 +65,7 @@
6065
import java.io.IOException;
6166
import java.util.Arrays;
6267
import java.util.List;
68+
import java.util.Map;
6369
import java.util.concurrent.ExecutionException;
6470
import javax.annotation.Nonnull;
6571

@@ -519,6 +525,52 @@ private ApiFuture<Table> getTableAsync(
519525
return transformToTableResponse(this.stub.getTableCallable().futureCall(request));
520526
}
521527

528+
/**
529+
* Gets the current encryption info for the table across all of the clusters.
530+
*
531+
* <p>The returned Map will be keyed by cluster id and contain a status for all of the keys in
532+
* use.
533+
*/
534+
public Map<String, List<EncryptionInfo>> getEncryptionInfo(String tableId) {
535+
return ApiExceptions.callAndTranslateApiException(getEncryptionInfoAsync(tableId));
536+
}
537+
538+
/**
539+
* Asynchronously gets the current encryption info for the table across all of the clusters.
540+
*
541+
* <p>The returned Map will be keyed by cluster id and contain a status for all of the keys in
542+
* use.
543+
*/
544+
public ApiFuture<Map<String, List<EncryptionInfo>>> getEncryptionInfoAsync(String tableId) {
545+
GetTableRequest request =
546+
GetTableRequest.newBuilder()
547+
.setName(getTableName(tableId))
548+
.setView(View.ENCRYPTION_VIEW)
549+
.build();
550+
return ApiFutures.transform(
551+
this.stub.getTableCallable().futureCall(request),
552+
new ApiFunction<com.google.bigtable.admin.v2.Table, Map<String, List<EncryptionInfo>>>() {
553+
@Override
554+
public Map<String, List<EncryptionInfo>> apply(com.google.bigtable.admin.v2.Table table) {
555+
ImmutableMap.Builder<String, List<EncryptionInfo>> result = ImmutableMap.builder();
556+
557+
for (Map.Entry<String, ClusterState> entry : table.getClusterStatesMap().entrySet()) {
558+
ImmutableList.Builder<EncryptionInfo> infos = ImmutableList.builder();
559+
560+
for (com.google.bigtable.admin.v2.EncryptionInfo infoProto :
561+
entry.getValue().getEncryptionInfoList()) {
562+
infos.add(EncryptionInfo.fromProto(infoProto));
563+
}
564+
565+
result.put(entry.getKey(), infos.build());
566+
}
567+
568+
return result.build();
569+
}
570+
},
571+
MoreExecutors.directExecutor());
572+
}
573+
522574
/**
523575
* Lists all table IDs in the instance.
524576
*

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/Backup.java

+12
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,18 @@ public State getState() {
142142
return State.fromProto(proto.getState());
143143
}
144144

145+
/**
146+
* Get the encryption information for the backup.
147+
*
148+
* <p>If encryption_type is CUSTOMER_MANAGED_ENCRYPTION, kms_key_version will be filled in with
149+
* status UNKNOWN.
150+
*
151+
* <p>If encryption_type is GOOGLE_DEFAULT_ENCRYPTION, all other fields will have default value.
152+
*/
153+
public EncryptionInfo getEncryptionInfo() {
154+
return EncryptionInfo.fromProto(proto.getEncryptionInfo());
155+
}
156+
145157
@Override
146158
public boolean equals(Object o) {
147159
if (this == o) {

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/Cluster.java

+13
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
* in the instance.
3131
*/
3232
public class Cluster {
33+
3334
public enum State {
3435
/** The state of the cluster could not be determined. */
3536
NOT_KNOWN(com.google.bigtable.admin.v2.Cluster.State.STATE_NOT_KNOWN),
@@ -156,6 +157,18 @@ public StorageType getStorageType() {
156157
return StorageType.fromProto(stateProto.getDefaultStorageType());
157158
}
158159

160+
/**
161+
* Google Cloud Key Management Service (KMS) settings for a CMEK-protected Bigtable cluster. This
162+
* returns the full resource name of the Cloud KMS key in the format
163+
* `projects/{key_project_id}/locations/{location}/keyRings/{ring_name}/cryptoKeys/{key_name}`
164+
*/
165+
public String getKmsKeyName() {
166+
if (stateProto.hasEncryptionConfig()) {
167+
return stateProto.getEncryptionConfig().getKmsKeyName();
168+
}
169+
return null;
170+
}
171+
159172
@Override
160173
public boolean equals(Object o) {
161174
if (this == o) {

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/CreateClusterRequest.java

+12
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
* details</a>
5050
*/
5151
public final class CreateClusterRequest {
52+
5253
private final com.google.bigtable.admin.v2.CreateClusterRequest.Builder proto =
5354
com.google.bigtable.admin.v2.CreateClusterRequest.newBuilder();
5455
// instanceId and zone are short ids, which will be expanded to full names when the project name
@@ -104,6 +105,17 @@ public CreateClusterRequest setStorageType(@Nonnull StorageType storageType) {
104105
return this;
105106
}
106107

108+
/**
109+
* Sets the Google Cloud Key Management Service (KMS) key for a CMEK-protected Bigtable. This
110+
* requires the full resource name of the Cloud KMS key, in the format
111+
* `projects/{key_project_id}/locations/{location}/keyRings/{ring_name}/cryptoKeys/{key_name}`
112+
*/
113+
public CreateClusterRequest setKmsKeyName(@Nonnull String kmsKeyName) {
114+
Preconditions.checkNotNull(kmsKeyName);
115+
proto.getClusterBuilder().getEncryptionConfigBuilder().setKmsKeyName(kmsKeyName);
116+
return this;
117+
}
118+
107119
/**
108120
* Creates the request protobuf. This method is considered an internal implementation detail and
109121
* not meant to be used by applications.

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/CreateInstanceRequest.java

+29
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,35 @@ public CreateInstanceRequest addCluster(
147147
return this;
148148
}
149149

150+
/**
151+
* Adds a CMEK protected cluster using the specified KMS key name.
152+
*
153+
* @param clusterId the name of the cluster.
154+
* @param zone the zone where the cluster will be created.
155+
* @param serveNodes the number of nodes that cluster will contain. DEVELOPMENT instance clusters
156+
* must have exactly one node.
157+
* @param storageType the type of storage used by this cluster to serve its parent instance's
158+
* tables.
159+
* @param kmsKeyName the full name of the KMS key name to use in the format
160+
* `projects/{key_project_id}/locations/{location}/keyRings/{ring_name}/cryptoKeys/{key_name}`
161+
*/
162+
public CreateInstanceRequest addCmekCluster(
163+
@Nonnull String clusterId,
164+
@Nonnull String zone,
165+
int serveNodes,
166+
@Nonnull StorageType storageType,
167+
@Nonnull String kmsKeyName) {
168+
CreateClusterRequest clusterRequest =
169+
CreateClusterRequest.of("ignored-instance-id", clusterId)
170+
.setZone(zone)
171+
.setServeNodes(serveNodes)
172+
.setStorageType(storageType)
173+
.setKmsKeyName(kmsKeyName);
174+
clusterRequests.add(clusterRequest);
175+
176+
return this;
177+
}
178+
150179
/**
151180
* Adds a DEVELOPMENT cluster to the instance request.
152181
*
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
/*
2+
* Copyright 2021 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.bigtable.admin.v2.models;
17+
18+
import com.google.bigtable.admin.v2.EncryptionInfo.EncryptionType;
19+
import com.google.cloud.bigtable.common.Status;
20+
import com.google.common.base.Objects;
21+
22+
/**
23+
* Encryption information for a given resource.
24+
*
25+
* <p>If this resource is protected with customer managed encryption, the in-use Google Cloud Key
26+
* Management Service (KMS) key versions will be specified along with their status.
27+
*/
28+
public final class EncryptionInfo {
29+
public enum Type {
30+
/** Encryption type was not specified, though data at rest remains encrypted. */
31+
ENCRYPTION_TYPE_UNSPECIFIED(
32+
com.google.bigtable.admin.v2.EncryptionInfo.EncryptionType.ENCRYPTION_TYPE_UNSPECIFIED),
33+
/**
34+
* The data backing this resource is encrypted at rest with a key that is fully managed by
35+
* Google. No key version or status will be populated.
36+
*/
37+
GOOGLE_DEFAULT_ENCRYPTION(
38+
com.google.bigtable.admin.v2.EncryptionInfo.EncryptionType.GOOGLE_DEFAULT_ENCRYPTION),
39+
/**
40+
* The data backing this resource is encrypted at rest with a key that is fully managed by
41+
* Google. No key version or status will be populated. This is the default state.
42+
*/
43+
CUSTOMER_MANAGED_ENCRYPTION(
44+
com.google.bigtable.admin.v2.EncryptionInfo.EncryptionType.CUSTOMER_MANAGED_ENCRYPTION),
45+
/** Type not known by the client, please upgrade your client */
46+
UNRECOGNIZED(com.google.bigtable.admin.v2.EncryptionInfo.EncryptionType.UNRECOGNIZED);
47+
48+
private final com.google.bigtable.admin.v2.EncryptionInfo.EncryptionType proto;
49+
50+
Type(EncryptionType proto) {
51+
this.proto = proto;
52+
}
53+
54+
/** Wraps the EncryptionInfo protobuf. */
55+
public static Type fromProto(com.google.bigtable.admin.v2.EncryptionInfo.EncryptionType proto) {
56+
for (Type type : values()) {
57+
if (Objects.equal(type.proto, proto)) {
58+
return type;
59+
}
60+
}
61+
return UNRECOGNIZED;
62+
}
63+
}
64+
65+
private com.google.bigtable.admin.v2.EncryptionInfo proto;
66+
67+
public static EncryptionInfo fromProto(com.google.bigtable.admin.v2.EncryptionInfo proto) {
68+
return new EncryptionInfo(proto);
69+
}
70+
71+
private EncryptionInfo(com.google.bigtable.admin.v2.EncryptionInfo proto) {
72+
this.proto = proto;
73+
}
74+
75+
public Type getType() {
76+
return EncryptionInfo.Type.fromProto(proto.getEncryptionType());
77+
}
78+
79+
public String getKmsKeyVersion() {
80+
return proto.getKmsKeyVersion();
81+
}
82+
83+
public Status getStatus() {
84+
return Status.fromProto(proto.getEncryptionStatus());
85+
}
86+
87+
@Override
88+
public boolean equals(Object o) {
89+
if (this == o) {
90+
return true;
91+
}
92+
if (o == null || getClass() != o.getClass()) {
93+
return false;
94+
}
95+
EncryptionInfo that = (EncryptionInfo) o;
96+
return Objects.equal(proto, that.proto);
97+
}
98+
99+
@Override
100+
public int hashCode() {
101+
return Objects.hashCode(proto);
102+
}
103+
104+
@Override
105+
public String toString() {
106+
return proto.toString();
107+
}
108+
}

0 commit comments

Comments
 (0)