Skip to content

Commit ddb7391

Browse files
michaelpri10michaelpri10
authored
docs: Add ingestion from GCS sample (#2211)
* docs: Add ingestion from GCS Java sample * docs: Add ingestion from GCS sample * docs: Add test for GCS ingestion * docs: Move topicName declaration to satisfy style check * docs: Update bucket for Cloud Storage ingestion sample test
1 parent d938709 commit ddb7391

File tree

2 files changed

+150
-7
lines changed

2 files changed

+150
-7
lines changed

‎samples/snippets/src/main/java/pubsub/CreateTopicWithCloudStorageIngestionExample.java

+109
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
/*
2+
* Copyright 2024 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package pubsub;
18+
19+
// [START pubsub_create_topic_with_cloud_storage_ingestion]
20+
21+
import com.google.cloud.pubsub.v1.TopicAdminClient;
22+
import com.google.protobuf.util.Timestamps;
23+
import com.google.pubsub.v1.IngestionDataSourceSettings;
24+
import com.google.pubsub.v1.Topic;
25+
import com.google.pubsub.v1.TopicName;
26+
import java.io.IOException;
27+
import java.text.ParseException;
28+
29+
public class CreateTopicWithCloudStorageIngestionExample {
30+
public static void main(String... args) throws Exception {
31+
// TODO(developer): Replace these variables before running the sample.
32+
String projectId = "your-project-id";
33+
String topicId = "your-topic-id";
34+
// Cloud Storage ingestion settings.
35+
// bucket and inputFormat are required arguments.
36+
String bucket = "your-bucket";
37+
String inputFormat = "text";
38+
String textDelimiter = "\n";
39+
String matchGlob = "**.txt";
40+
String minimumObjectCreateTime = "YYYY-MM-DDThh:mm:ssZ";
41+
42+
createTopicWithCloudStorageIngestionExample(
43+
projectId, topicId, bucket, inputFormat, textDelimiter, matchGlob, minimumObjectCreateTime);
44+
}
45+
46+
public static void createTopicWithCloudStorageIngestionExample(
47+
String projectId,
48+
String topicId,
49+
String bucket,
50+
String inputFormat,
51+
String textDelimiter,
52+
String matchGlob,
53+
String minimumObjectCreateTime)
54+
throws IOException {
55+
try (TopicAdminClient topicAdminClient = TopicAdminClient.create()) {
56+
IngestionDataSourceSettings.CloudStorage.Builder cloudStorageBuilder =
57+
IngestionDataSourceSettings.CloudStorage.newBuilder().setBucket(bucket);
58+
switch (inputFormat) {
59+
case "text":
60+
cloudStorageBuilder.setTextFormat(
61+
IngestionDataSourceSettings.CloudStorage.TextFormat.newBuilder()
62+
.setDelimiter(textDelimiter)
63+
.build());
64+
break;
65+
case "avro":
66+
cloudStorageBuilder.setAvroFormat(
67+
IngestionDataSourceSettings.CloudStorage.AvroFormat.getDefaultInstance());
68+
break;
69+
case "pubsub_avro":
70+
cloudStorageBuilder.setPubsubAvroFormat(
71+
IngestionDataSourceSettings.CloudStorage.PubSubAvroFormat.getDefaultInstance());
72+
break;
73+
default:
74+
throw new IllegalArgumentException(
75+
"inputFormat must be in ('text', 'avro', 'pubsub_avro'); got value: " + inputFormat);
76+
}
77+
78+
if (matchGlob != null && !matchGlob.isEmpty()) {
79+
cloudStorageBuilder.setMatchGlob(matchGlob);
80+
}
81+
82+
if (minimumObjectCreateTime != null && !minimumObjectCreateTime.isEmpty()) {
83+
try {
84+
cloudStorageBuilder.setMinimumObjectCreateTime(Timestamps.parse(minimumObjectCreateTime));
85+
} catch (ParseException e) {
86+
System.err.println("Unable to parse timestamp: " + minimumObjectCreateTime);
87+
}
88+
}
89+
90+
IngestionDataSourceSettings ingestionDataSourceSettings =
91+
IngestionDataSourceSettings.newBuilder()
92+
.setCloudStorage(cloudStorageBuilder.build())
93+
.build();
94+
95+
TopicName topicName = TopicName.of(projectId, topicId);
96+
97+
Topic topic =
98+
topicAdminClient.createTopic(
99+
Topic.newBuilder()
100+
.setName(topicName.toString())
101+
.setIngestionDataSourceSettings(ingestionDataSourceSettings)
102+
.build());
103+
104+
System.out.println(
105+
"Created topic with Cloud Storage ingestion settings: " + topic.getAllFields());
106+
}
107+
}
108+
}
109+
// [END pubsub_create_topic_with_cloud_storage_ingestion]

‎samples/snippets/src/test/java/pubsub/AdminIT.java

+41-7
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,9 @@ public class AdminIT {
5252
private static final String projectId = System.getenv("GOOGLE_CLOUD_PROJECT");
5353
private static final String _suffix = UUID.randomUUID().toString();
5454
private static final String topicId = "iam-topic-" + _suffix;
55-
private static final String ingestionTopicId = "ingestion-topic-" + _suffix;
55+
private static final String kinesisIngestionTopicId = "kinesis-ingestion-topic-" + _suffix;
56+
private static final String cloudStorageIngestionTopicId =
57+
"cloud-storage-ingestion-topic-" + _suffix;
5658
private static final String pullSubscriptionId = "iam-pull-subscription-" + _suffix;
5759
private static final String pushSubscriptionId = "iam-push-subscription-" + _suffix;
5860
private static final String orderedSubscriptionId = "iam-ordered-subscription-" + _suffix;
@@ -75,9 +77,18 @@ public class AdminIT {
7577
private static final String awsRoleArn = "arn:aws:iam::111111111111:role/fake-role-name";
7678
private static final String gcpServiceAccount =
7779
"[email protected]";
80+
private static final String cloudStorageBucket = "pubsub-cloud-storage-bucket";
81+
private static final String cloudStorageInputFormat = "text";
82+
private static final String cloudStorageTextDelimiter = ",";
83+
private static final String cloudStorageMatchGlob = "**.txt";
84+
private static final String cloudStorageMinimumObjectCreateTime = "1970-01-01T00:00:00Z";
85+
private static final String cloudStorageMinimumObjectCreateTimeSeconds = "0";
7886

7987
private static final TopicName topicName = TopicName.of(projectId, topicId);
80-
private static final TopicName ingestionTopicName = TopicName.of(projectId, ingestionTopicId);
88+
private static final TopicName kinesisIngestionTopicName =
89+
TopicName.of(projectId, kinesisIngestionTopicId);
90+
private static final TopicName cloudStorageIngestionTopicName =
91+
TopicName.of(projectId, cloudStorageIngestionTopicId);
8192
private static final SubscriptionName pullSubscriptionName =
8293
SubscriptionName.of(projectId, pullSubscriptionId);
8394
private static final SubscriptionName pushSubscriptionName =
@@ -304,9 +315,9 @@ public void testAdmin() throws Exception {
304315
bout.reset();
305316
// Test create topic with Kinesis ingestion settings.
306317
CreateTopicWithKinesisIngestionExample.createTopicWithKinesisIngestionExample(
307-
projectId, ingestionTopicId, streamArn, consumerArn, awsRoleArn, gcpServiceAccount);
318+
projectId, kinesisIngestionTopicId, streamArn, consumerArn, awsRoleArn, gcpServiceAccount);
308319
assertThat(bout.toString())
309-
.contains("google.pubsub.v1.Topic.name=" + ingestionTopicName.toString());
320+
.contains("google.pubsub.v1.Topic.name=" + kinesisIngestionTopicName.toString());
310321
assertThat(bout.toString()).contains(streamArn);
311322
assertThat(bout.toString()).contains(consumerArn);
312323
assertThat(bout.toString()).contains(awsRoleArn);
@@ -315,17 +326,40 @@ public void testAdmin() throws Exception {
315326
bout.reset();
316327
// Test update existing Kinesis ingestion settings.
317328
UpdateTopicTypeExample.updateTopicTypeExample(
318-
projectId, ingestionTopicId, streamArn, consumerArn2, awsRoleArn, gcpServiceAccount);
329+
projectId, kinesisIngestionTopicId, streamArn, consumerArn2, awsRoleArn, gcpServiceAccount);
319330
assertThat(bout.toString())
320-
.contains("google.pubsub.v1.Topic.name=" + ingestionTopicName.toString());
331+
.contains("google.pubsub.v1.Topic.name=" + kinesisIngestionTopicName.toString());
321332
assertThat(bout.toString()).contains(streamArn);
322333
assertThat(bout.toString()).contains(consumerArn2);
323334
assertThat(bout.toString()).contains(awsRoleArn);
324335
assertThat(bout.toString()).contains(gcpServiceAccount);
325336

326337
bout.reset();
327338
// Test delete Kinesis ingestion topic.
328-
DeleteTopicExample.deleteTopicExample(projectId, ingestionTopicId);
339+
DeleteTopicExample.deleteTopicExample(projectId, kinesisIngestionTopicId);
340+
assertThat(bout.toString()).contains("Deleted topic.");
341+
342+
bout.reset();
343+
// Test create topic with Cloud Storage ingestion settings.
344+
CreateTopicWithCloudStorageIngestionExample.createTopicWithCloudStorageIngestionExample(
345+
projectId,
346+
cloudStorageIngestionTopicId,
347+
cloudStorageBucket,
348+
cloudStorageInputFormat,
349+
cloudStorageTextDelimiter,
350+
cloudStorageMatchGlob,
351+
cloudStorageMinimumObjectCreateTime);
352+
assertThat(bout.toString())
353+
.contains("google.pubsub.v1.Topic.name=" + cloudStorageIngestionTopicName.toString());
354+
assertThat(bout.toString()).contains(cloudStorageBucket);
355+
assertThat(bout.toString()).contains(cloudStorageInputFormat);
356+
assertThat(bout.toString()).contains(cloudStorageTextDelimiter);
357+
assertThat(bout.toString()).contains(cloudStorageMatchGlob);
358+
assertThat(bout.toString()).contains(cloudStorageMinimumObjectCreateTimeSeconds);
359+
360+
bout.reset();
361+
// Test delete Cloud Storage ingestion topic.
362+
DeleteTopicExample.deleteTopicExample(projectId, cloudStorageIngestionTopicId);
329363
assertThat(bout.toString()).contains("Deleted topic.");
330364
}
331365
}

0 commit comments

Comments
 (0)