diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000..e43d91c0ba --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,10 @@ +# Code owners file. +# This file controls who is tagged for review for any given pull request. +# +# For syntax help see: +# https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax + + +# The bigtable-dpe team is the default owner for anything not +# explicitly taken by someone else. +* @googleapis/bigtable-dpe diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000000..445b4bf82f --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,76 @@ +on: + push: + branches: + - master + pull_request: +name: ci +jobs: + units: + runs-on: ubuntu-latest + strategy: + matrix: + java: [7, 8, 11] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: ${{matrix.java}} + - run: java -version + - run: .kokoro/build.sh + env: + JOB_TYPE: test + - name: coverage + uses: codecov/codecov-action@v1 + with: + name: actions ${{matrix.java}} + windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - run: java -version + - run: .kokoro/build.bat + env: + JOB_TYPE: test + dependencies: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - run: java -version + - run: .kokoro/dependencies.sh + linkage-monitor: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - run: java -version + - run: .kokoro/linkage-monitor.sh + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - run: java -version + - run: .kokoro/build.sh + env: + JOB_TYPE: lint + clirr: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - run: java -version + - run: .kokoro/build.sh + env: + JOB_TYPE: clirr \ No newline at end of file diff --git a/.kokoro/build.bat b/.kokoro/build.bat index 65b013ab16..05826ad93f 100644 --- a/.kokoro/build.bat +++ b/.kokoro/build.bat @@ -1,3 +1,3 @@ :: See documentation in type-shell-output.bat -"C:\Program Files\Git\bin\bash.exe" github/java-bigtable/.kokoro/build.sh +"C:\Program Files\Git\bin\bash.exe" %~dp0build.sh diff --git a/.repo-metadata.json b/.repo-metadata.json index 96bffe6296..b6e5527352 100644 --- a/.repo-metadata.json +++ b/.repo-metadata.json @@ -2,7 +2,7 @@ "name": "bigtable", "name_pretty": "Cloud Bigtable", "product_documentation": "https://cloud.google.com/bigtable", - "client_documentation": "https://googleapis.dev/java/java-bigtable/latest/index.html", + "client_documentation": "https://googleapis.dev/java/google-cloud-bigtable/latest/index.html", "issue_tracker": "https://issuetracker.google.com/savedsearches/559777", "release_level": "ga", "language": "java", diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a79e36c97..30b6e594eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,34 @@ # Changelog +## [1.13.0](https://www.github.com/googleapis/java-bigtable/compare/v1.12.2...v1.13.0) (2020-05-27) + + +### Features + +* adding utility to transform protobuf into model object ([#299](https://www.github.com/googleapis/java-bigtable/issues/299)) ([00f6d2d](https://www.github.com/googleapis/java-bigtable/commit/00f6d2da4179eb3f4f55a1fe1da04047697c5999)) +* expose new API with ReadRowsRequest in EnhancedBigtableStub ([#276](https://www.github.com/googleapis/java-bigtable/issues/276)) ([394efe4](https://www.github.com/googleapis/java-bigtable/commit/394efe459ebe34030bf1d09116eebb4ec4f311e9)) +* Update opencensus metrics to include bigtable resource ids and rpc level metrics ([#214](https://www.github.com/googleapis/java-bigtable/issues/214)) ([7214ef6](https://www.github.com/googleapis/java-bigtable/commit/7214ef6853fc6892401b55bd1beeccbe896e4f33)) + + +### Bug Fixes + +* bigtable v2 retry config settings to disable streaming RPC retries ([#315](https://www.github.com/googleapis/java-bigtable/issues/315)) ([a7e43f0](https://www.github.com/googleapis/java-bigtable/commit/a7e43f07dc0b93c6702992d45444e815200bf202)) +* change app_profile_id to the correct header app_profile ([#318](https://www.github.com/googleapis/java-bigtable/issues/318)) ([237b15d](https://www.github.com/googleapis/java-bigtable/commit/237b15d14b420005d32f227e1a76e7d308db0a42)) + + +### Dependencies + +* update autovalue.version to v1.7.2 ([#306](https://www.github.com/googleapis/java-bigtable/issues/306)) ([bc215c5](https://www.github.com/googleapis/java-bigtable/commit/bc215c5883b16727c28893b4a13b3387f0a04ac9)) +* update dependency com.google.api:api-common to v1.9.1 ([#316](https://www.github.com/googleapis/java-bigtable/issues/316)) ([482603a](https://www.github.com/googleapis/java-bigtable/commit/482603acd4c3db461b0e5f489a8c55213b9f3326)) +* update dependency com.google.cloud:google-cloud-conformance-tests to v0.0.11 ([#307](https://www.github.com/googleapis/java-bigtable/issues/307)) ([50a19a6](https://www.github.com/googleapis/java-bigtable/commit/50a19a6aded1b313390ff78b52d9ffc4f4cc7fdd)) +* update dependency com.google.cloud:google-cloud-core-bom to v1.93.5 ([#317](https://www.github.com/googleapis/java-bigtable/issues/317)) ([7f61f90](https://www.github.com/googleapis/java-bigtable/commit/7f61f903abc16554c88850061faf03e6b8beb3b9)) +* update protobuf.version to v3.12.1 ([#308](https://www.github.com/googleapis/java-bigtable/issues/308)) ([99e3a95](https://www.github.com/googleapis/java-bigtable/commit/99e3a95d456bb79841720e8ecbdcb0d47017535e)) + + +### Documentation + +* **fix:** update client documentation link ([#312](https://www.github.com/googleapis/java-bigtable/issues/312)) ([7e04c7a](https://www.github.com/googleapis/java-bigtable/commit/7e04c7a69a80d71303ce62b4320153facd3cfad8)) + ### [1.12.2](https://www.github.com/googleapis/java-bigtable/compare/v1.12.1...v1.12.2) (2020-05-11) diff --git a/README.md b/README.md index 4cfbec85cd..fbfd62da1c 100644 --- a/README.md +++ b/README.md @@ -16,16 +16,16 @@ If you are using Maven, add this to your pom.xml file com.google.cloud google-cloud-bigtable - 1.12.2 + 1.13.0 ``` If you are using Gradle, add this to your dependencies ```Groovy -compile 'com.google.cloud:google-cloud-bigtable:1.12.2' +compile 'com.google.cloud:google-cloud-bigtable:1.13.0' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "1.12.2" +libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "1.13.0" ``` [//]: # ({x-version-update-end}) @@ -176,13 +176,13 @@ If you are using Maven, add this to your pom.xml file io.opencensus opencensus-impl - 0.24.0 + 0.26.0 runtime io.opencensus opencensus-exporter-trace-stackdriver - 0.24.0 + 0.26.0 io.grpc @@ -197,13 +197,13 @@ If you are using Maven, add this to your pom.xml file ``` If you are using Gradle, add this to your dependencies ```Groovy -compile 'io.opencensus:opencensus-impl:0.24.0' -compile 'io.opencensus:opencensus-exporter-trace-stackdriver:0.24.0' +compile 'io.opencensus:opencensus-impl:0.26.0' +compile 'io.opencensus:opencensus-exporter-trace-stackdriver:0.26.0' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "io.opencensus" % "opencensus-impl" % "0.24.0" -libraryDependencies += "io.opencensus" % "opencensus-exporter-trace-stackdriver" % "0.24.0" +libraryDependencies += "io.opencensus" % "opencensus-impl" % "0.26.0" +libraryDependencies += "io.opencensus" % "opencensus-exporter-trace-stackdriver" % "0.26.0" ``` At the start of your application configure the exporter: @@ -236,30 +236,37 @@ Tracing.getTraceConfig().updateActiveTraceParams( Cloud Bigtable client supports [Opencensus Metrics](https://opencensus.io/stats/), which gives insight into the client internals and aids in debugging production issues. -Metrics prefixed with `cloud.google.com/java/bigtable/` focus on operation level -metrics across all of the retry attempts that occurred during that operation. RPC -level metrics can be gleaned from gRPC's metrics, which are prefixed with -`grpc.io/client/`. +All Cloud Bigtable Metrics are prefixed with `cloud.google.com/java/bigtable/`. The +metrics will be tagged with: + * `bigtable_project_id`: the project that contains the target Bigtable instance. + Please note that this id could be different from project that the client is running + in and different from the project where the metrics are exported to. +* `bigtable_instance_id`: the instance id of the target Bigtable instance +* `bigtable_app_profile_id`: the app profile id that is being used to access the target + Bigtable instance ### Available operation level metric views: -* `cloud.google.com/java/bigtable/op_latency`: A distribution latency of +* `cloud.google.com/java/bigtable/op_latency`: A distribution of latency of each client method call, across all of it's RPC attempts. Tagged by - method name and final response status. + operation name and final response status. * `cloud.google.com/java/bigtable/completed_ops`: The total count of - method invocations. Tagged by method name. Can be compared to - `grpc.io/client/completed_rpcs` to visualize retry attempts. + method invocations. Tagged by operation name and final response status. * `cloud.google.com/java/bigtable/read_rows_first_row_latency`: A distribution of the latency of receiving the first row in a ReadRows operation. -* `cloud.google.com/java/bigtable/rows_per_op`: A distribution of rows - read per ReadRows operation across all retry attempts. +* `cloud.google.com/java/bigtable/attempt_latency`: A distribution of latency of + each client RPC, tagged by operation name and the attempt status. Under normal + circumstances, this will be identical to op_latency. However, when the client + receives transient errors, op_latency will be the sum of all attempt_latencies + and the exponential delays -* `cloud.google.com/java/bigtable/mutations_per_batch`: A distribution - of mutations per BulkMutation. +* `cloud.google.com/java/bigtable/attempts_per_op`: A distribution of attempts that + each operation required, tagged by operation name and final operation status. + Under normal circumstances, this will be 1. By default, the functionality is disabled. For example to enable metrics using @@ -273,13 +280,13 @@ If you are using Maven, add this to your pom.xml file io.opencensus opencensus-impl - 0.24.0 + 0.26.0 runtime io.opencensus opencensus-exporter-stats-stackdriver - 0.24.0 + 0.26.0 io.grpc @@ -294,13 +301,13 @@ If you are using Maven, add this to your pom.xml file ``` If you are using Gradle, add this to your dependencies ```Groovy -compile 'io.opencensus:opencensus-impl:0.24.0' -compile 'io.opencensus:opencensus-exporter-stats-stackdriver:0.24.0' +compile 'io.opencensus:opencensus-impl:0.26.0' +compile 'io.opencensus:opencensus-exporter-stats-stackdriver:0.26.0' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "io.opencensus" % "opencensus-impl" % "0.24.0" -libraryDependencies += "io.opencensus" % "opencensus-exporter-stats-stackdriver" % "0.24.0" +libraryDependencies += "io.opencensus" % "opencensus-impl" % "0.26.0" +libraryDependencies += "io.opencensus" % "opencensus-exporter-stats-stackdriver" % "0.26.0" ``` At the start of your application configure the exporter and enable the Bigtable stats views: @@ -337,7 +344,7 @@ Add the following to your project's pom.xml. io.grpc grpc-bom - 1.27.0 + 1.28.0 pom import diff --git a/google-cloud-bigtable-bom/pom.xml b/google-cloud-bigtable-bom/pom.xml index f345024f81..370f6cccec 100644 --- a/google-cloud-bigtable-bom/pom.xml +++ b/google-cloud-bigtable-bom/pom.xml @@ -3,12 +3,12 @@ 4.0.0 com.google.cloud google-cloud-bigtable-bom - 1.12.2 + 1.13.0 pom com.google.cloud google-cloud-shared-config - 0.5.0 + 0.6.0 Google Cloud Bigtable BOM @@ -72,32 +72,32 @@ com.google.cloud google-cloud-bigtable - 1.12.2 + 1.13.0 com.google.cloud google-cloud-bigtable-emulator - 0.121.2 + 0.122.0 com.google.api.grpc grpc-google-cloud-bigtable-admin-v2 - 1.12.2 + 1.13.0 com.google.api.grpc grpc-google-cloud-bigtable-v2 - 1.12.2 + 1.13.0 com.google.api.grpc proto-google-cloud-bigtable-admin-v2 - 1.12.2 + 1.13.0 com.google.api.grpc proto-google-cloud-bigtable-v2 - 1.12.2 + 1.13.0 diff --git a/google-cloud-bigtable-deps-bom/pom.xml b/google-cloud-bigtable-deps-bom/pom.xml index e8a0eee81f..3b071292b8 100644 --- a/google-cloud-bigtable-deps-bom/pom.xml +++ b/google-cloud-bigtable-deps-bom/pom.xml @@ -7,12 +7,12 @@ com.google.cloud google-cloud-shared-config - 0.5.0 + 0.6.0 com.google.cloud google-cloud-bigtable-deps-bom - 1.12.2 + 1.13.0 pom @@ -71,19 +71,19 @@ - 1.7.1 + 1.7.2 1.56.0 - 1.9.0 + 1.9.1 1.18.0 - 1.93.4 + 1.93.5 0.20.0 1.35.0 1.29.0 29.0-android 0.26.0 - 3.11.4 + 3.12.1 1.4.4 @@ -190,6 +190,11 @@ opencensus-api ${opencensus.version} + + io.opencensus + opencensus-impl-core + ${opencensus.version} + io.opencensus opencensus-contrib-grpc-util diff --git a/google-cloud-bigtable-emulator/pom.xml b/google-cloud-bigtable-emulator/pom.xml index 3576992f97..9846bc7e10 100644 --- a/google-cloud-bigtable-emulator/pom.xml +++ b/google-cloud-bigtable-emulator/pom.xml @@ -5,7 +5,7 @@ 4.0.0 google-cloud-bigtable-emulator - 0.121.2 + 0.122.0 Google Cloud Java - Bigtable Emulator https://github.com/googleapis/java-bigtable @@ -14,7 +14,7 @@ com.google.cloud google-cloud-bigtable-parent - 1.12.2 + 1.13.0 scm:git:git@github.com:googleapis/java-bigtable.git @@ -80,14 +80,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.12.2 + 1.13.0 pom import com.google.cloud google-cloud-bigtable-bom - 1.12.2 + 1.13.0 pom import diff --git a/google-cloud-bigtable/clirr-ignored-differences.xml b/google-cloud-bigtable/clirr-ignored-differences.xml index 9dec1aa01d..ab921a973f 100644 --- a/google-cloud-bigtable/clirr-ignored-differences.xml +++ b/google-cloud-bigtable/clirr-ignored-differences.xml @@ -7,4 +7,20 @@ *snapshot* *snapshot* + + 8001 + com/google/cloud/bigtable/data/v2/stub/metrics/MeasuredMutateRowsCallable* + + + 8001 + com/google/cloud/bigtable/data/v2/stub/metrics/MeasuredReadRowsCallable* + + + 8001 + com/google/cloud/bigtable/data/v2/stub/metrics/MeasuredUnaryCallable* + + + 8001 + com/google/cloud/bigtable/gaxx/tracing/WrappedTracerFactory* + \ No newline at end of file diff --git a/google-cloud-bigtable/pom.xml b/google-cloud-bigtable/pom.xml index c4c7609845..d33c079274 100644 --- a/google-cloud-bigtable/pom.xml +++ b/google-cloud-bigtable/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-bigtable - 1.12.2 + 1.13.0 jar Google Cloud Bigtable https://github.com/googleapis/java-bigtable @@ -12,7 +12,7 @@ com.google.cloud google-cloud-bigtable-parent - 1.12.2 + 1.13.0 google-cloud-bigtable @@ -36,14 +36,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.12.2 + 1.13.0 pom import com.google.cloud google-cloud-bigtable-bom - 1.12.2 + 1.13.0 pom import @@ -172,12 +172,6 @@ org.threeten threetenbp - - - - - - @@ -218,6 +212,11 @@ grpc-testing test + + io.opencensus + opencensus-impl + test + junit junit @@ -385,8 +384,12 @@ io.grpc:grpc-auth,io.grpc:grpc-grpclb,com.google.auto.value:auto-value + + io.opencensus:opencensus-impl-core + diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/BulkMutation.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/BulkMutation.java index fb45e8e458..95d10872ec 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/BulkMutation.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/BulkMutation.java @@ -15,6 +15,7 @@ */ package com.google.cloud.bigtable.data.v2.models; +import com.google.api.core.BetaApi; import com.google.api.core.InternalApi; import com.google.bigtable.v2.MutateRowsRequest; import com.google.cloud.bigtable.data.v2.internal.NameUtil; @@ -114,6 +115,26 @@ public MutateRowsRequest toProto(RequestContext requestContext) { .build(); } + /** + * Wraps the protobuf {@link MutateRowsRequest}. + * + *

This is meant for advanced usage only. Please ensure that the MutateRowsRequest does not use + * server side timestamps. The BigtableDataClient assumes that mutation present in BulkMutation + * are idempotent and is configured to enable retries by default. If serverside timestamps are + * enabled then that can lead to duplicate mutations. + * + *

WARNING: when applied, the resulting mutation object will ignore the project id and instance + * id in the table_name and instead apply the configuration in the client. + */ + @BetaApi + public static BulkMutation fromProto(@Nonnull MutateRowsRequest request) { + BulkMutation bulkMutation = + BulkMutation.create(NameUtil.extractTableIdFromTableName(request.getTableName())); + bulkMutation.builder = request.toBuilder(); + + return bulkMutation; + } + /** Creates a copy of {@link BulkMutation}. */ @Override public BulkMutation clone() { diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutation.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutation.java index 15cc2be1f4..ac4c548942 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutation.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutation.java @@ -15,6 +15,7 @@ */ package com.google.cloud.bigtable.data.v2.models; +import com.google.api.core.BetaApi; import com.google.api.core.InternalApi; import com.google.bigtable.v2.CheckAndMutateRowRequest; import com.google.cloud.bigtable.data.v2.internal.NameUtil; @@ -136,4 +137,20 @@ public CheckAndMutateRowRequest toProto(RequestContext requestContext) { .setAppProfileId(requestContext.getAppProfileId()) .build(); } + + /** + * Wraps the protobuf {@link CheckAndMutateRowRequest}. + * + *

WARNING: Please note that the table_name will be overwritten by the configuration in the + * BigtableDataClient. + */ + @BetaApi + public static ConditionalRowMutation fromProto(@Nonnull CheckAndMutateRowRequest request) { + String tableId = NameUtil.extractTableIdFromTableName(request.getTableName()); + ConditionalRowMutation rowMutation = + ConditionalRowMutation.create(tableId, request.getRowKey()); + rowMutation.builder = request.toBuilder(); + + return rowMutation; + } } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/Mutation.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/Mutation.java index 2a6b655ee4..326d78cfe0 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/Mutation.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/Mutation.java @@ -87,6 +87,22 @@ public static Mutation fromProtoUnsafe(List pro return mutation; } + /** + * Constructs a row mutation from an existing protobuf object. + * + *

Callers must ensure that the protobuf argument is not using serverside timestamps. The + * client assumes that all mutations are idempotent and will retry in case of transient errors. + * This can lead to row duplication. + * + *

When applied, the resulting Mutation object will ignore the project id and instance id in + * the table_name and instead apply the configuration in the client + */ + static Mutation fromProto(List protos) { + Mutation mutation = new Mutation(false); + mutation.mutations.addAll(protos); + return mutation; + } + private Mutation(boolean allowServersideTimestamp) { this.allowServersideTimestamp = allowServersideTimestamp; } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRow.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRow.java index 26253b9cbc..5fa483d1bd 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRow.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRow.java @@ -15,6 +15,7 @@ */ package com.google.cloud.bigtable.data.v2.models; +import com.google.api.core.BetaApi; import com.google.api.core.InternalApi; import com.google.bigtable.v2.ReadModifyWriteRowRequest; import com.google.bigtable.v2.ReadModifyWriteRule; @@ -137,4 +138,20 @@ public ReadModifyWriteRowRequest toProto(RequestContext requestContext) { .setAppProfileId(requestContext.getAppProfileId()) .build(); } + + /** + * Wraps the protobuf {@link ReadModifyWriteRowRequest}. + * + *

WARNING: Please note that the table_name will be overwritten by the configuration in the + * BigtableDataClient. + */ + @BetaApi + public static ReadModifyWriteRow fromProto(@Nonnull ReadModifyWriteRowRequest request) { + String tableId = NameUtil.extractTableIdFromTableName(request.getTableName()); + + ReadModifyWriteRow row = ReadModifyWriteRow.create(tableId, request.getRowKey()); + row.builder = request.toBuilder(); + + return row; + } } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/RowMutation.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/RowMutation.java index 37b658e8c9..6c5456aac3 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/RowMutation.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/RowMutation.java @@ -15,6 +15,7 @@ */ package com.google.cloud.bigtable.data.v2.models; +import com.google.api.core.BetaApi; import com.google.api.core.InternalApi; import com.google.bigtable.v2.MutateRowRequest; import com.google.bigtable.v2.MutateRowsRequest; @@ -214,4 +215,23 @@ public MutateRowsRequest toBulkProto(RequestContext requestContext) { Entry.newBuilder().setRowKey(key).addAllMutations(mutation.getMutations()).build()) .build(); } + + /** + * Wraps the protobuf {@link MutateRowRequest}. + * + *

This is meant for advanced usage only. Please ensure that the MutateRowRequest does not use + * server side timestamps. The BigtableDataClient assumes that RowMutations are idempotent and is + * configured to enable retries by default. If serverside timestamps are enabled, this can lead to + * duplicate mutations. + * + *

WARNING: when applied, the resulting mutation object will ignore the project id and instance + * id in the table_name and instead apply the configuration in the client. + */ + @BetaApi + public static RowMutation fromProto(@Nonnull MutateRowRequest request) { + String tableId = NameUtil.extractTableIdFromTableName(request.getTableName()); + + return RowMutation.create( + tableId, request.getRowKey(), Mutation.fromProto(request.getMutationsList())); + } } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableStubSettings.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableStubSettings.java index a34bbd6734..f7660e9528 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableStubSettings.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableStubSettings.java @@ -311,13 +311,13 @@ private static Builder initDefaults(Builder builder) { builder .readRowsSettings() - .setRetryableCodes(RETRYABLE_CODE_DEFINITIONS.get("idempotent")) + .setRetryableCodes(RETRYABLE_CODE_DEFINITIONS.get("non_idempotent")) .setRetrySettings(RETRY_PARAM_DEFINITIONS.get("read_rows_params")); builder .sampleRowKeysSettings() - .setRetryableCodes(RETRYABLE_CODE_DEFINITIONS.get("idempotent")) - .setRetrySettings(RETRY_PARAM_DEFINITIONS.get("idempotent_params")); + .setRetryableCodes(RETRYABLE_CODE_DEFINITIONS.get("non_idempotent")) + .setRetrySettings(RETRY_PARAM_DEFINITIONS.get("non_idempotent_params")); builder .mutateRowSettings() @@ -326,7 +326,7 @@ private static Builder initDefaults(Builder builder) { builder .mutateRowsSettings() - .setRetryableCodes(RETRYABLE_CODE_DEFINITIONS.get("idempotent")) + .setRetryableCodes(RETRYABLE_CODE_DEFINITIONS.get("non_idempotent")) .setRetrySettings(RETRY_PARAM_DEFINITIONS.get("mutate_rows_params")); builder diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java index 4b5ede9da9..4de6643472 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java @@ -15,10 +15,13 @@ */ package com.google.cloud.bigtable.data.v2.stub; +import com.google.api.core.BetaApi; import com.google.api.core.InternalApi; import com.google.api.gax.batching.Batcher; import com.google.api.gax.batching.BatcherImpl; import com.google.api.gax.core.BackgroundResource; +import com.google.api.gax.core.GaxProperties; +import com.google.api.gax.grpc.GaxGrpcProperties; import com.google.api.gax.grpc.GrpcCallSettings; import com.google.api.gax.grpc.GrpcRawCallableFactory; import com.google.api.gax.retrying.ExponentialRetryAlgorithm; @@ -31,6 +34,7 @@ import com.google.api.gax.rpc.ServerStreamingCallSettings; import com.google.api.gax.rpc.ServerStreamingCallable; import com.google.api.gax.rpc.UnaryCallable; +import com.google.api.gax.tracing.OpencensusTracerFactory; import com.google.api.gax.tracing.SpanName; import com.google.api.gax.tracing.TracedServerStreamingCallable; import com.google.api.gax.tracing.TracedUnaryCallable; @@ -58,9 +62,9 @@ import com.google.cloud.bigtable.data.v2.models.RowAdapter; import com.google.cloud.bigtable.data.v2.models.RowMutation; import com.google.cloud.bigtable.data.v2.models.RowMutationEntry; -import com.google.cloud.bigtable.data.v2.stub.metrics.MeasuredMutateRowsCallable; -import com.google.cloud.bigtable.data.v2.stub.metrics.MeasuredReadRowsCallable; -import com.google.cloud.bigtable.data.v2.stub.metrics.MeasuredUnaryCallable; +import com.google.cloud.bigtable.data.v2.stub.metrics.CompositeTracerFactory; +import com.google.cloud.bigtable.data.v2.stub.metrics.MetricsTracerFactory; +import com.google.cloud.bigtable.data.v2.stub.metrics.RpcMeasureConstants; import com.google.cloud.bigtable.data.v2.stub.mutaterows.BulkMutateRowsUserFacingCallable; import com.google.cloud.bigtable.data.v2.stub.mutaterows.MutateRowsBatchingDescriptor; import com.google.cloud.bigtable.data.v2.stub.mutaterows.MutateRowsRetryingCallable; @@ -72,10 +76,13 @@ import com.google.cloud.bigtable.data.v2.stub.readrows.RowMergingCallable; import com.google.cloud.bigtable.gaxx.retrying.ApiResultRetryAlgorithm; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.protobuf.ByteString; import io.opencensus.stats.Stats; import io.opencensus.stats.StatsRecorder; +import io.opencensus.tags.TagKey; +import io.opencensus.tags.TagValue; import io.opencensus.tags.Tagger; import io.opencensus.tags.Tags; import java.io.IOException; @@ -97,16 +104,12 @@ */ @InternalApi public class EnhancedBigtableStub implements AutoCloseable { - private static final String TRACING_OUTER_CLIENT_NAME = "Bigtable"; + private static final String CLIENT_NAME = "Bigtable"; private final EnhancedBigtableStubSettings settings; private final ClientContext clientContext; private final RequestContext requestContext; - // TODO: This should probably move to ClientContext - private final Tagger tagger; - private final StatsRecorder statsRecorder; - private final ServerStreamingCallable readRowsCallable; private final UnaryCallable readRowCallable; private final UnaryCallable> sampleRowKeysCallable; @@ -124,15 +127,58 @@ public static EnhancedBigtableStub create(EnhancedBigtableStubSettings settings) } @InternalApi("Visible for testing") - private EnhancedBigtableStub( + public EnhancedBigtableStub( EnhancedBigtableStubSettings settings, ClientContext clientContext, Tagger tagger, StatsRecorder statsRecorder) { this.settings = settings; - this.clientContext = clientContext; - this.tagger = tagger; - this.statsRecorder = statsRecorder; + + this.clientContext = + clientContext + .toBuilder() + .setTracerFactory( + new CompositeTracerFactory( + ImmutableList.of( + // Add OpenCensus Tracing + new OpencensusTracerFactory( + ImmutableMap.builder() + // Annotate traces with the same tags as metrics + .put( + RpcMeasureConstants.BIGTABLE_PROJECT_ID.getName(), + settings.getProjectId()) + .put( + RpcMeasureConstants.BIGTABLE_INSTANCE_ID.getName(), + settings.getInstanceId()) + .put( + RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID.getName(), + settings.getAppProfileId()) + // Also annotate traces with library versions + .put("gax", GaxGrpcProperties.getGaxGrpcVersion()) + .put("grpc", GaxGrpcProperties.getGrpcVersion()) + .put( + "gapic", + GaxProperties.getLibraryVersion( + EnhancedBigtableStubSettings.class)) + .build()), + // Add OpenCensus Metrics + MetricsTracerFactory.create( + tagger, + statsRecorder, + ImmutableMap.builder() + .put( + RpcMeasureConstants.BIGTABLE_PROJECT_ID, + TagValue.create(settings.getProjectId())) + .put( + RpcMeasureConstants.BIGTABLE_INSTANCE_ID, + TagValue.create(settings.getInstanceId())) + .put( + RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID, + TagValue.create(settings.getAppProfileId())) + .build()), + // Add user configured tracer + clientContext.getTracerFactory()))) + .build(); this.requestContext = RequestContext.create( settings.getProjectId(), settings.getInstanceId(), settings.getAppProfileId()); @@ -148,6 +194,27 @@ private EnhancedBigtableStub( // + /** + * Creates a callable chain to handle ReadRows RPCs. The chain will: + * + *

    + *
  • Dispatch the RPC with {@link ReadRowsRequest}. + *
  • Upon receiving the response stream, it will merge the {@link + * com.google.bigtable.v2.ReadRowsResponse.CellChunk}s in logical rows. The actual row + * implementation can be configured by the {@code rowAdapter} parameter. + *
  • Retry/resume on failure. + *
  • Filter out marker rows. + *
+ * + *

NOTE: the caller is responsible for adding tracing & metrics. + */ + @BetaApi("This surface is stable yet it might be removed in the future.") + public ServerStreamingCallable createReadRowsRawCallable( + RowAdapter rowAdapter) { + return createReadRowsBaseCallable(settings.readRowsSettings(), rowAdapter) + .withDefaultCallContext(clientContext.getDefaultCallContext()); + } + /** * Creates a callable chain to handle streaming ReadRows RPCs. The chain will: * @@ -164,24 +231,19 @@ private EnhancedBigtableStub( */ public ServerStreamingCallable createReadRowsCallable( RowAdapter rowAdapter) { - ServerStreamingCallable readRowsCallable = + ServerStreamingCallable readRowsCallable = createReadRowsBaseCallable(settings.readRowsSettings(), rowAdapter); + ServerStreamingCallable readRowsUserCallable = + new ReadRowsUserCallable<>(readRowsCallable, requestContext); + ServerStreamingCallable traced = new TracedServerStreamingCallable<>( - readRowsCallable, + readRowsUserCallable, clientContext.getTracerFactory(), - SpanName.of(TRACING_OUTER_CLIENT_NAME, "ReadRows")); - - ServerStreamingCallable measured = - new MeasuredReadRowsCallable<>( - traced, - TRACING_OUTER_CLIENT_NAME + ".ReadRows", - tagger, - statsRecorder, - clientContext.getClock()); + SpanName.of(CLIENT_NAME, "ReadRows")); - return measured.withDefaultCallContext(clientContext.getDefaultCallContext()); + return traced.withDefaultCallContext(clientContext.getDefaultCallContext()); } /** @@ -199,15 +261,17 @@ public ServerStreamingCallable createReadRowsCallable( * */ public UnaryCallable createReadRowCallable(RowAdapter rowAdapter) { - UnaryCallable readRowCallable = + ServerStreamingCallable readRowsCallable = createReadRowsBaseCallable( - ServerStreamingCallSettings.newBuilder() - .setRetryableCodes(settings.readRowSettings().getRetryableCodes()) - .setRetrySettings(settings.readRowSettings().getRetrySettings()) - .setIdleTimeout(settings.readRowSettings().getRetrySettings().getTotalTimeout()) - .build(), - rowAdapter) - .first(); + ServerStreamingCallSettings.newBuilder() + .setRetryableCodes(settings.readRowSettings().getRetryableCodes()) + .setRetrySettings(settings.readRowSettings().getRetrySettings()) + .setIdleTimeout(settings.readRowSettings().getRetrySettings().getTotalTimeout()) + .build(), + rowAdapter); + + UnaryCallable readRowCallable = + new ReadRowsUserCallable<>(readRowsCallable, requestContext).first(); return createUserFacingUnaryCallable("ReadRow", readRowCallable); } @@ -216,19 +280,18 @@ public UnaryCallable createReadRowCallable(RowAdapter * Creates a callable chain to handle ReadRows RPCs. The chain will: * *

    - *
  • Convert a {@link Query} into a {@link com.google.bigtable.v2.ReadRowsRequest} and - * dispatch the RPC. + *
  • Dispatch the RPC with {@link ReadRowsRequest}. *
  • Upon receiving the response stream, it will merge the {@link * com.google.bigtable.v2.ReadRowsResponse.CellChunk}s in logical rows. The actual row - * implementation can be configured in by the {@code rowAdapter} parameter. + * implementation can be configured by the {@code rowAdapter} parameter. *
  • Retry/resume on failure. *
  • Filter out marker rows. *
* *

NOTE: the caller is responsible for adding tracing & metrics. */ - private ServerStreamingCallable createReadRowsBaseCallable( - ServerStreamingCallSettings readRowsSettings, RowAdapter rowAdapter) { + private ServerStreamingCallable createReadRowsBaseCallable( + ServerStreamingCallSettings readRowsSettings, RowAdapter rowAdapter) { ServerStreamingCallable base = GrpcRawCallableFactory.createServerStreamingCallable( @@ -240,7 +303,7 @@ private ServerStreamingCallable createReadRowsBaseCallable( public Map extract(ReadRowsRequest readRowsRequest) { return ImmutableMap.of( "table_name", readRowsRequest.getTableName(), - "app_profile_id", readRowsRequest.getAppProfileId()); + "app_profile", readRowsRequest.getAppProfileId()); } }) .build(), @@ -249,8 +312,8 @@ public Map extract(ReadRowsRequest readRowsRequest) { ServerStreamingCallable merging = new RowMergingCallable<>(base, rowAdapter); - // Copy settings for the middle ReadRowsRequest -> RowT callable (as opposed to the outer - // Query -> RowT callable or the inner ReadRowsRequest -> ReadRowsResponse callable). + // Copy settings for the middle ReadRowsRequest -> RowT callable (as opposed to the inner + // ReadRowsRequest -> ReadRowsResponse callable). ServerStreamingCallSettings innerSettings = ServerStreamingCallSettings.newBuilder() .setResumptionStrategy(new ReadRowsResumptionStrategy<>(rowAdapter)) @@ -270,10 +333,7 @@ public Map extract(ReadRowsRequest readRowsRequest) { ServerStreamingCallable retrying2 = Callables.retrying(retrying1, innerSettings, clientContext); - FilterMarkerRowsCallable filtering = - new FilterMarkerRowsCallable<>(retrying2, rowAdapter); - - return new ReadRowsUserCallable<>(filtering, requestContext); + return new FilterMarkerRowsCallable<>(retrying2, rowAdapter); } /** @@ -300,7 +360,7 @@ public Map extract( SampleRowKeysRequest sampleRowKeysRequest) { return ImmutableMap.of( "table_name", sampleRowKeysRequest.getTableName(), - "app_profile_id", sampleRowKeysRequest.getAppProfileId()); + "app_profile", sampleRowKeysRequest.getAppProfileId()); } }) .build(), @@ -334,7 +394,7 @@ private UnaryCallable createMutateRowCallable() { public Map extract(MutateRowRequest mutateRowRequest) { return ImmutableMap.of( "table_name", mutateRowRequest.getTableName(), - "app_profile_id", mutateRowRequest.getAppProfileId()); + "app_profile", mutateRowRequest.getAppProfileId()); } }) .build(), @@ -370,19 +430,9 @@ private UnaryCallable createBulkMutateRowsCallable() { UnaryCallable traced = new TracedUnaryCallable<>( - userFacing, - clientContext.getTracerFactory(), - SpanName.of(TRACING_OUTER_CLIENT_NAME, "MutateRows")); - - UnaryCallable measured = - new MeasuredMutateRowsCallable( - traced, - TRACING_OUTER_CLIENT_NAME + ".MutateRows", - tagger, - statsRecorder, - clientContext.getClock()); + userFacing, clientContext.getTracerFactory(), SpanName.of(CLIENT_NAME, "MutateRows")); - return measured.withDefaultCallContext(clientContext.getDefaultCallContext()); + return traced.withDefaultCallContext(clientContext.getDefaultCallContext()); } /** @@ -457,7 +507,7 @@ private UnaryCallable createMutateRowsBaseCallable() { public Map extract(MutateRowsRequest mutateRowsRequest) { return ImmutableMap.of( "table_name", mutateRowsRequest.getTableName(), - "app_profile_id", mutateRowsRequest.getAppProfileId()); + "app_profile", mutateRowsRequest.getAppProfileId()); } }) .build(), @@ -499,7 +549,7 @@ public Map extract( CheckAndMutateRowRequest checkAndMutateRowRequest) { return ImmutableMap.of( "table_name", checkAndMutateRowRequest.getTableName(), - "app_profile_id", checkAndMutateRowRequest.getAppProfileId()); + "app_profile", checkAndMutateRowRequest.getAppProfileId()); } }) .build(), @@ -533,7 +583,7 @@ private UnaryCallable createReadModifyWriteRowCallable( public Map extract(ReadModifyWriteRowRequest request) { return ImmutableMap.of( "table_name", request.getTableName(), - "app_profile_id", request.getAppProfileId()); + "app_profile", request.getAppProfileId()); } }) .build(), @@ -555,19 +605,9 @@ private UnaryCallable createUserFacin UnaryCallable traced = new TracedUnaryCallable<>( - inner, - clientContext.getTracerFactory(), - SpanName.of(TRACING_OUTER_CLIENT_NAME, methodName)); - - UnaryCallable measured = - new MeasuredUnaryCallable<>( - traced, - TRACING_OUTER_CLIENT_NAME + "." + methodName, - tagger, - statsRecorder, - clientContext.getClock()); + inner, clientContext.getTracerFactory(), SpanName.of(CLIENT_NAME, methodName)); - return measured.withDefaultCallContext(clientContext.getDefaultCallContext()); + return traced.withDefaultCallContext(clientContext.getDefaultCallContext()); } // diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java index ce8b4bb2b1..139e7c2233 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java @@ -20,9 +20,7 @@ import com.google.api.gax.batching.BatchingSettings; import com.google.api.gax.batching.FlowControlSettings; import com.google.api.gax.batching.FlowController.LimitExceededBehavior; -import com.google.api.gax.core.GaxProperties; import com.google.api.gax.core.GoogleCredentialsProvider; -import com.google.api.gax.grpc.GaxGrpcProperties; import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; import com.google.api.gax.retrying.RetrySettings; import com.google.api.gax.rpc.ServerStreamingCallSettings; @@ -30,7 +28,6 @@ import com.google.api.gax.rpc.StubSettings; import com.google.api.gax.rpc.TransportChannelProvider; import com.google.api.gax.rpc.UnaryCallSettings; -import com.google.api.gax.tracing.OpencensusTracerFactory; import com.google.cloud.bigtable.data.v2.internal.RefreshChannel; import com.google.cloud.bigtable.data.v2.models.ConditionalRowMutation; import com.google.cloud.bigtable.data.v2.models.KeyOffset; @@ -42,7 +39,6 @@ import com.google.cloud.bigtable.data.v2.stub.readrows.ReadRowsBatchingDescriptor; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Set; @@ -522,13 +518,6 @@ private Builder() { setInternalHeaderProvider( BigtableStubSettings.defaultApiClientHeaderProviderBuilder().build()); - setTracerFactory( - new OpencensusTracerFactory( - ImmutableMap.of( - "gax", GaxGrpcProperties.getGaxGrpcVersion(), - "grpc", GaxGrpcProperties.getGrpcVersion(), - "gapic", GaxProperties.getLibraryVersion(EnhancedBigtableStubSettings.class)))); - // Per-method settings using baseSettings for defaults. readRowsSettings = ServerStreamingCallSettings.newBuilder(); diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/CompositeTracer.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/CompositeTracer.java new file mode 100644 index 0000000000..33bf9c42b9 --- /dev/null +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/CompositeTracer.java @@ -0,0 +1,154 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.stub.metrics; + +import com.google.api.gax.tracing.ApiTracer; +import com.google.common.collect.ImmutableList; +import java.util.ArrayList; +import java.util.List; +import org.threeten.bp.Duration; + +/** Combines multiple {@link ApiTracer}s into a single {@link ApiTracer}. */ +class CompositeTracer implements ApiTracer { + private final List children; + + CompositeTracer(List children) { + this.children = ImmutableList.copyOf(children); + } + + @Override + public Scope inScope() { + final List childScopes = new ArrayList<>(children.size()); + + for (ApiTracer child : children) { + childScopes.add(child.inScope()); + } + + return new Scope() { + @Override + public void close() { + for (Scope childScope : childScopes) { + childScope.close(); + } + } + }; + } + + @Override + public void operationSucceeded() { + for (ApiTracer child : children) { + child.operationSucceeded(); + } + } + + @Override + public void operationCancelled() { + for (ApiTracer child : children) { + child.operationCancelled(); + } + } + + @Override + public void operationFailed(Throwable error) { + for (ApiTracer child : children) { + child.operationFailed(error); + } + } + + @Override + public void connectionSelected(String id) { + for (ApiTracer child : children) { + child.connectionSelected(id); + } + } + + @Override + public void attemptStarted(int attemptNumber) { + for (ApiTracer child : children) { + child.attemptStarted(attemptNumber); + } + } + + @Override + public void attemptSucceeded() { + for (ApiTracer child : children) { + child.attemptSucceeded(); + } + } + + @Override + public void attemptCancelled() { + for (ApiTracer child : children) { + child.attemptCancelled(); + } + } + + @Override + public void attemptFailed(Throwable error, Duration delay) { + for (ApiTracer child : children) { + child.attemptFailed(error, delay); + } + } + + @Override + public void attemptFailedRetriesExhausted(Throwable error) { + for (ApiTracer child : children) { + child.attemptFailedRetriesExhausted(error); + } + } + + @Override + public void attemptPermanentFailure(Throwable error) { + for (ApiTracer child : children) { + child.attemptPermanentFailure(error); + } + } + + @Override + public void lroStartFailed(Throwable error) { + for (ApiTracer child : children) { + child.lroStartFailed(error); + } + } + + @Override + public void lroStartSucceeded() { + for (ApiTracer child : children) { + child.lroStartSucceeded(); + } + } + + @Override + public void responseReceived() { + for (ApiTracer child : children) { + child.responseReceived(); + } + } + + @Override + public void requestSent() { + for (ApiTracer child : children) { + child.requestSent(); + } + } + + @Override + public void batchRequestSent(long elementCount, long requestSize) { + for (ApiTracer child : children) { + child.batchRequestSent(elementCount, requestSize); + } + } +} diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/tracing/WrappedTracerFactory.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/CompositeTracerFactory.java similarity index 50% rename from google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/tracing/WrappedTracerFactory.java rename to google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/CompositeTracerFactory.java index 253d7a207a..e2e399ae3d 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/gaxx/tracing/WrappedTracerFactory.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/CompositeTracerFactory.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Google LLC + * Copyright 2020 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,36 +13,32 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.cloud.bigtable.gaxx.tracing; +package com.google.cloud.bigtable.data.v2.stub.metrics; import com.google.api.core.InternalApi; import com.google.api.gax.tracing.ApiTracer; import com.google.api.gax.tracing.ApiTracerFactory; import com.google.api.gax.tracing.SpanName; +import com.google.common.collect.ImmutableList; +import java.util.ArrayList; +import java.util.List; -/** - * Simple wrapper around {@link ApiTracerFactory} to augment the client name of the generated - * traces. - * - *

This is used to disambiguate traces in underlying GAPIC client from the manually written - * overlay. - * - *

For internal use, public for technical reasons. - */ -@InternalApi -public class WrappedTracerFactory implements ApiTracerFactory { - private final ApiTracerFactory innerFactory; - private final String clientName; +/** Combines multiple {@link ApiTracerFactory} into a single {@link ApiTracerFactory}. */ +@InternalApi("For internal use only") +public class CompositeTracerFactory implements ApiTracerFactory { + private final List apiTracerFactories; - public WrappedTracerFactory(ApiTracerFactory tracerFactory, String clientName) { - this.innerFactory = tracerFactory; - this.clientName = clientName; + public CompositeTracerFactory(List apiTracerFactories) { + this.apiTracerFactories = ImmutableList.copyOf(apiTracerFactories); } @Override public ApiTracer newTracer(ApiTracer parent, SpanName spanName, OperationType operationType) { - spanName = SpanName.of(clientName, spanName.getMethodName()); + List children = new ArrayList<>(apiTracerFactories.size()); - return innerFactory.newTracer(parent, spanName, operationType); + for (ApiTracerFactory factory : apiTracerFactories) { + children.add(factory.newTracer(parent, spanName, operationType)); + } + return new CompositeTracer(children); } } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MeasuredMutateRowsCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MeasuredMutateRowsCallable.java deleted file mode 100644 index 11878635fe..0000000000 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MeasuredMutateRowsCallable.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.cloud.bigtable.data.v2.stub.metrics; - -import com.google.api.core.ApiClock; -import com.google.api.core.ApiFuture; -import com.google.api.core.InternalApi; -import com.google.api.gax.rpc.ApiCallContext; -import com.google.api.gax.rpc.UnaryCallable; -import com.google.cloud.bigtable.data.v2.models.BulkMutation; -import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.MoreExecutors; -import io.opencensus.stats.StatsRecorder; -import io.opencensus.tags.TagContext; -import io.opencensus.tags.TagValue; -import io.opencensus.tags.Tagger; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import javax.annotation.Nonnull; - -/** - * This callable will instrument MutateRows invocations using OpenCensus stats. - * - *

Recorded stats: - * - *

- *
{@link RpcMeasureConstants#BIGTABLE_OP_LATENCY} - *
the total time it took the operation across all of its retry attempts to complete. - *
{@link RpcMeasureConstants#BIGTABLE_MUTATE_ROWS_ENTRIES_PER_BATCH} - *
the number of mutations sent per batch operation. Retry attempts might have few entries. - *
- * - *

For internal use only. - */ -@InternalApi -public class MeasuredMutateRowsCallable extends UnaryCallable { - private final UnaryCallable innerCallable; - private final TagValue methodName; - private final TagContext parentCtx; - private final Tagger tagger; - private final StatsRecorder stats; - private final ApiClock clock; - - @InternalApi - public MeasuredMutateRowsCallable( - @Nonnull UnaryCallable innerCallable, - @Nonnull String methodName, - @Nonnull Tagger tagger, - @Nonnull StatsRecorder stats, - @Nonnull ApiClock clock) { - this.innerCallable = Preconditions.checkNotNull(innerCallable, "innerCallable"); - this.methodName = TagValue.create(Preconditions.checkNotNull(methodName, "methodName")); - this.tagger = Preconditions.checkNotNull(tagger, "tagger"); - this.parentCtx = tagger.getCurrentTagContext(); - this.stats = Preconditions.checkNotNull(stats, "stats"); - this.clock = Preconditions.checkNotNull(clock, "clock"); - } - - @Override - public ApiFuture futureCall(BulkMutation request, ApiCallContext context) { - long operationStartTime = clock.nanoTime(); - - final ApiFuture future = innerCallable.futureCall(request, context); - future.addListener( - new StatsRecordingRunnable(future, operationStartTime, request.getEntryCount()), - MoreExecutors.directExecutor()); - return future; - } - - private class StatsRecordingRunnable implements Runnable { - private final Future operationFuture; - private final long operationStart; - private final long numEntries; - - private StatsRecordingRunnable( - @Nonnull Future operationFuture, long operationStartTime, long numEntries) { - this.operationFuture = Preconditions.checkNotNull(operationFuture, "operationFuture"); - this.operationStart = operationStartTime; - this.numEntries = numEntries; - } - - @Override - public void run() { - long elapsed = TimeUnit.NANOSECONDS.toMillis(clock.nanoTime() - operationStart); - - stats - .newMeasureMap() - .put(RpcMeasureConstants.BIGTABLE_OP_LATENCY, elapsed) - .put(RpcMeasureConstants.BIGTABLE_MUTATE_ROWS_ENTRIES_PER_BATCH, numEntries) - .record( - tagger - .toBuilder(parentCtx) - .putLocal(RpcMeasureConstants.BIGTABLE_OP, methodName) - .putLocal( - RpcMeasureConstants.BIGTABLE_STATUS, Util.extractStatus(operationFuture)) - .build()); - } - } -} diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MeasuredReadRowsCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MeasuredReadRowsCallable.java deleted file mode 100644 index 0ac3777eab..0000000000 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MeasuredReadRowsCallable.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.cloud.bigtable.data.v2.stub.metrics; - -import com.google.api.core.ApiClock; -import com.google.api.core.InternalApi; -import com.google.api.gax.rpc.ApiCallContext; -import com.google.api.gax.rpc.ResponseObserver; -import com.google.api.gax.rpc.ServerStreamingCallable; -import com.google.api.gax.rpc.StreamController; -import com.google.cloud.bigtable.data.v2.models.Query; -import com.google.common.base.Preconditions; -import io.opencensus.stats.MeasureMap; -import io.opencensus.stats.StatsRecorder; -import io.opencensus.tags.TagContext; -import io.opencensus.tags.TagValue; -import io.opencensus.tags.Tagger; -import java.util.concurrent.TimeUnit; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -/** - * This callable will instrument ReadRows invocations using OpenCensus stats. - * - *

Recorded stats: - * - *

- *
{@link RpcMeasureConstants#BIGTABLE_OP_LATENCY} - *
the total time it took the operation across all of its retry attempts to complete. - *
{@link RpcMeasureConstants#BIGTABLE_ROWS_READ_PER_OP} - *
the number of rows received across all of the retries for each invocation. - *
{@link RpcMeasureConstants#BIGTABLE_READ_ROWS_FIRST_ROW_LATENCY} - *
the amount of time it took the caller to receive the first row. - *
- * - *

For internal use only. - */ -@InternalApi -public class MeasuredReadRowsCallable extends ServerStreamingCallable { - private final ServerStreamingCallable innerCallable; - - private final TagValue methodName; - private final TagContext parentCtx; - - private final Tagger tagger; - private final StatsRecorder stats; - private final ApiClock clock; - - public MeasuredReadRowsCallable( - @Nonnull ServerStreamingCallable innerCallable, - @Nonnull String methodName, - @Nonnull Tagger tagger, - @Nonnull StatsRecorder stats, - @Nonnull ApiClock clock) { - this.innerCallable = Preconditions.checkNotNull(innerCallable, "innerCallable"); - this.methodName = TagValue.create(Preconditions.checkNotNull(methodName, "methodName")); - this.tagger = Preconditions.checkNotNull(tagger, "tagger"); - this.parentCtx = tagger.getCurrentTagContext(); - this.stats = Preconditions.checkNotNull(stats, "stats"); - this.clock = Preconditions.checkNotNull(clock, "clock"); - } - - @Override - public void call(Query request, ResponseObserver outerObserver, ApiCallContext context) { - innerCallable.call(request, new MeasuredResponseObserver(outerObserver), context); - } - - private class MeasuredResponseObserver implements ResponseObserver { - private final ResponseObserver outerResponseObserver; - - private final long operationStart; - private Long firstRowReceivedAt = null; - private long rowsRead = 0; - - private MeasuredResponseObserver(@Nonnull ResponseObserver outerResponseObserver) { - this.outerResponseObserver = - Preconditions.checkNotNull(outerResponseObserver, "outerResponseObserver"); - this.operationStart = clock.nanoTime(); - } - - @Override - public void onStart(StreamController controller) { - outerResponseObserver.onStart(controller); - } - - @Override - public void onResponse(RowT row) { - if (firstRowReceivedAt == null) { - firstRowReceivedAt = clock.nanoTime(); - } - rowsRead++; - outerResponseObserver.onResponse(row); - } - - @Override - public void onError(Throwable t) { - recordStats(t); - outerResponseObserver.onError(t); - } - - @Override - public void onComplete() { - recordStats(null); - outerResponseObserver.onComplete(); - } - - private void recordStats(@Nullable Throwable error) { - long now = clock.nanoTime(); - long elapsed = TimeUnit.NANOSECONDS.toMillis(now - operationStart); - - MeasureMap measures = - stats - .newMeasureMap() - .put(RpcMeasureConstants.BIGTABLE_OP_LATENCY, elapsed) - .put(RpcMeasureConstants.BIGTABLE_ROWS_READ_PER_OP, rowsRead); - - if (firstRowReceivedAt != null) { - long firstRowLatency = TimeUnit.NANOSECONDS.toMillis(firstRowReceivedAt - operationStart); - measures.put(RpcMeasureConstants.BIGTABLE_READ_ROWS_FIRST_ROW_LATENCY, firstRowLatency); - } - - measures.record( - tagger - .toBuilder(parentCtx) - .putLocal(RpcMeasureConstants.BIGTABLE_OP, methodName) - .putLocal(RpcMeasureConstants.BIGTABLE_STATUS, Util.extractStatus(error)) - .build()); - } - } -} diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MeasuredUnaryCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MeasuredUnaryCallable.java deleted file mode 100644 index aade43fd2c..0000000000 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MeasuredUnaryCallable.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.cloud.bigtable.data.v2.stub.metrics; - -import com.google.api.core.ApiClock; -import com.google.api.core.ApiFuture; -import com.google.api.core.InternalApi; -import com.google.api.gax.rpc.ApiCallContext; -import com.google.api.gax.rpc.UnaryCallable; -import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.MoreExecutors; -import io.opencensus.stats.StatsRecorder; -import io.opencensus.tags.TagContext; -import io.opencensus.tags.TagValue; -import io.opencensus.tags.Tagger; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import javax.annotation.Nonnull; - -/** - * This callable will instrument callable invocations using OpenCensus stats. - * - *

Recorded stats: - * - *

- *
{@link RpcMeasureConstants#BIGTABLE_OP_LATENCY} - *
the total time it took the operation across all of its retry attempts to complete - *
- * - *

For internal use only. - */ -@InternalApi -public class MeasuredUnaryCallable extends UnaryCallable { - private final UnaryCallable innerCallable; - - private final TagValue methodName; - private final TagContext parentCtx; - - private final Tagger tagger; - private final StatsRecorder stats; - private final ApiClock clock; - - public MeasuredUnaryCallable( - @Nonnull UnaryCallable innerCallable, - @Nonnull String methodName, - @Nonnull Tagger tagger, - @Nonnull StatsRecorder stats, - @Nonnull ApiClock clock) { - this.innerCallable = Preconditions.checkNotNull(innerCallable, "innerCallable"); - this.methodName = TagValue.create(Preconditions.checkNotNull(methodName, "methodName")); - this.tagger = Preconditions.checkNotNull(tagger, "tagger"); - this.parentCtx = tagger.getCurrentTagContext(); - this.stats = Preconditions.checkNotNull(stats, "stats"); - this.clock = Preconditions.checkNotNull(clock, "clock"); - } - - @Override - public ApiFuture futureCall(RequestT request, ApiCallContext context) { - long startTime = clock.nanoTime(); - ApiFuture future = innerCallable.futureCall(request, context); - future.addListener( - new StatsRecordingRunnable(future, startTime), MoreExecutors.directExecutor()); - return future; - } - - private class StatsRecordingRunnable implements Runnable { - private final Future operationFuture; - private final long operationStart; - - private StatsRecordingRunnable(@Nonnull Future operationFuture, long startTime) { - this.operationFuture = Preconditions.checkNotNull(operationFuture, "operationFuture"); - this.operationStart = startTime; - } - - @Override - public void run() { - long elapsed = TimeUnit.NANOSECONDS.toMillis(clock.nanoTime() - operationStart); - - stats - .newMeasureMap() - .put(RpcMeasureConstants.BIGTABLE_OP_LATENCY, elapsed) - .record( - tagger - .toBuilder(parentCtx) - .putLocal(RpcMeasureConstants.BIGTABLE_OP, methodName) - .putLocal( - RpcMeasureConstants.BIGTABLE_STATUS, Util.extractStatus(operationFuture)) - .build()); - } - } -} diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracer.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracer.java new file mode 100644 index 0000000000..864ba75020 --- /dev/null +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracer.java @@ -0,0 +1,218 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.stub.metrics; + +import com.google.api.gax.tracing.ApiTracer; +import com.google.api.gax.tracing.ApiTracerFactory.OperationType; +import com.google.api.gax.tracing.SpanName; +import com.google.common.base.Stopwatch; +import io.opencensus.stats.MeasureMap; +import io.opencensus.stats.StatsRecorder; +import io.opencensus.tags.TagContext; +import io.opencensus.tags.TagContextBuilder; +import io.opencensus.tags.TagKey; +import io.opencensus.tags.TagValue; +import io.opencensus.tags.Tagger; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.CancellationException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.annotation.Nullable; +import org.threeten.bp.Duration; + +class MetricsTracer implements ApiTracer { + private final OperationType operationType; + + private final Tagger tagger; + private final StatsRecorder stats; + + // Tags + private final TagContext parentContext; + private final SpanName spanName; + private final Map statsAttributes; + + // Operation level metrics + private final AtomicBoolean opFinished = new AtomicBoolean(); + private final Stopwatch operationTimer = Stopwatch.createStarted(); + private final Stopwatch firstResponsePerOpTimer = Stopwatch.createStarted(); + private long operationResponseCount = 0; + + // Attempt level metrics + private int attemptCount = 0; + private Stopwatch attemptTimer; + private long attemptResponseCount = 0; + + MetricsTracer( + OperationType operationType, + Tagger tagger, + StatsRecorder stats, + SpanName spanName, + Map statsAttributes) { + this.operationType = operationType; + this.tagger = tagger; + this.stats = stats; + this.parentContext = tagger.getCurrentTagContext(); + this.spanName = spanName; + this.statsAttributes = statsAttributes; + } + + @Override + public Scope inScope() { + return new Scope() { + @Override + public void close() {} + }; + } + + @Override + public void operationSucceeded() { + recordOperationCompletion(null); + } + + @Override + public void operationCancelled() { + recordOperationCompletion(new CancellationException()); + } + + @Override + public void operationFailed(Throwable throwable) { + recordOperationCompletion(throwable); + } + + private void recordOperationCompletion(@Nullable Throwable throwable) { + if (!opFinished.compareAndSet(false, true)) { + return; + } + operationTimer.stop(); + + long elapsed = operationTimer.elapsed(TimeUnit.MILLISECONDS); + + MeasureMap measures = + stats + .newMeasureMap() + .put(RpcMeasureConstants.BIGTABLE_OP_LATENCY, elapsed) + .put(RpcMeasureConstants.BIGTABLE_OP_ATTEMPT_COUNT, attemptCount); + + if (operationType == OperationType.ServerStreaming + && spanName.getMethodName().equals("ReadRows")) { + measures.put( + RpcMeasureConstants.BIGTABLE_READ_ROWS_FIRST_ROW_LATENCY, + firstResponsePerOpTimer.elapsed(TimeUnit.MILLISECONDS)); + } + + TagContextBuilder tagCtx = + newTagCtxBuilder() + .putLocal(RpcMeasureConstants.BIGTABLE_STATUS, Util.extractStatus(throwable)); + + measures.record(tagCtx.build()); + } + + @Override + public void connectionSelected(String s) { + // noop: cardinality for connection ids is too high to use as tags + } + + @Override + public void attemptStarted(int i) { + attemptCount++; + attemptTimer = Stopwatch.createStarted(); + attemptResponseCount = 0; + } + + @Override + public void attemptSucceeded() { + recordAttemptCompletion(null); + } + + @Override + public void attemptCancelled() { + recordAttemptCompletion(new CancellationException()); + } + + @Override + public void attemptFailed(Throwable throwable, Duration duration) { + recordAttemptCompletion(throwable); + } + + @Override + public void attemptFailedRetriesExhausted(Throwable throwable) { + recordAttemptCompletion(throwable); + } + + @Override + public void attemptPermanentFailure(Throwable throwable) { + recordAttemptCompletion(throwable); + } + + private void recordAttemptCompletion(@Nullable Throwable throwable) { + MeasureMap measures = + stats + .newMeasureMap() + .put( + RpcMeasureConstants.BIGTABLE_ATTEMPT_LATENCY, + attemptTimer.elapsed(TimeUnit.MILLISECONDS)); + + TagContextBuilder tagCtx = + newTagCtxBuilder() + .putLocal(RpcMeasureConstants.BIGTABLE_STATUS, Util.extractStatus(throwable)); + + measures.record(tagCtx.build()); + } + + @Override + public void lroStartFailed(Throwable throwable) { + // noop + } + + @Override + public void lroStartSucceeded() { + // noop + } + + @Override + public void responseReceived() { + if (firstResponsePerOpTimer.isRunning()) { + firstResponsePerOpTimer.stop(); + } + attemptResponseCount++; + operationResponseCount++; + } + + @Override + public void requestSent() { + // noop: no operations are client streaming + } + + @Override + public void batchRequestSent(long elementCount, long requestSize) { + // noop + } + + private TagContextBuilder newTagCtxBuilder() { + TagContextBuilder tagCtx = + tagger + .toBuilder(parentContext) + .putLocal(RpcMeasureConstants.BIGTABLE_OP, TagValue.create(spanName.toString())); + + // Copy client level tags in + for (Entry entry : statsAttributes.entrySet()) { + tagCtx.putLocal(entry.getKey(), entry.getValue()); + } + + return tagCtx; + } +} diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracerFactory.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracerFactory.java new file mode 100644 index 0000000000..24b22d3531 --- /dev/null +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracerFactory.java @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.stub.metrics; + +import com.google.api.core.InternalApi; +import com.google.api.gax.tracing.ApiTracer; +import com.google.api.gax.tracing.ApiTracerFactory; +import com.google.api.gax.tracing.SpanName; +import com.google.common.collect.ImmutableMap; +import io.opencensus.stats.StatsRecorder; +import io.opencensus.tags.TagKey; +import io.opencensus.tags.TagValue; +import io.opencensus.tags.Tagger; + +/** + * {@link ApiTracerFactory} that will generate OpenCensus metrics by using the {@link ApiTracer} + * api. + */ +@InternalApi("For internal use only") +public class MetricsTracerFactory implements ApiTracerFactory { + private final Tagger tagger; + private final StatsRecorder stats; + private final ImmutableMap statsAttributes; + + public static MetricsTracerFactory create( + Tagger tagger, StatsRecorder stats, ImmutableMap statsAttributes) { + return new MetricsTracerFactory(tagger, stats, statsAttributes); + } + + private MetricsTracerFactory( + Tagger tagger, StatsRecorder stats, ImmutableMap statsAttributes) { + this.tagger = tagger; + this.stats = stats; + this.statsAttributes = statsAttributes; + } + + @Override + public ApiTracer newTracer(ApiTracer parent, SpanName spanName, OperationType operationType) { + return new MetricsTracer(operationType, tagger, stats, spanName, statsAttributes); + } +} diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/RpcMeasureConstants.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/RpcMeasureConstants.java index f5830d05d3..8c6e347a0f 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/RpcMeasureConstants.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/RpcMeasureConstants.java @@ -15,53 +15,63 @@ */ package com.google.cloud.bigtable.data.v2.stub.metrics; -import io.opencensus.stats.Measure; -import io.opencensus.stats.Measure.MeasureDouble; +import com.google.api.core.InternalApi; import io.opencensus.stats.Measure.MeasureLong; import io.opencensus.tags.TagKey; -class RpcMeasureConstants { - /** - * Tag key that represents a Bigtable operation name. - * - *

A Bigtable operation consists of 1 or more RPCs. By comparing metrics tagged with {@link - * io.opencensus.contrib.grpc.metrics.RpcMeasureConstants#GRPC_CLIENT_METHOD} to methods tagged - * with {@link RpcMeasureConstants#BIGTABLE_OP}, the end user can get a sense how many attempts an - * operation took. - */ - public static final TagKey BIGTABLE_OP = TagKey.create("bigtable_op"); +@InternalApi("For internal use only") +public class RpcMeasureConstants { + // TagKeys + public static final TagKey BIGTABLE_PROJECT_ID = TagKey.create("bigtable_project_id"); + public static final TagKey BIGTABLE_INSTANCE_ID = TagKey.create("bigtable_instance_id"); + public static final TagKey BIGTABLE_APP_PROFILE_ID = TagKey.create("bigtable_app_profile_id"); + + /** Tag key that represents a Bigtable operation name. */ + static final TagKey BIGTABLE_OP = TagKey.create("bigtable_op"); /** Tag key that represents the final status of the Bigtable operation. */ - public static final TagKey BIGTABLE_STATUS = TagKey.create("bigtable_status"); + static final TagKey BIGTABLE_STATUS = TagKey.create("bigtable_status"); + // Units /** Unit to represent counts. */ private static final String COUNT = "1"; /** Unit to represent milliseconds. */ private static final String MILLISECOND = "ms"; - static final MeasureDouble BIGTABLE_OP_LATENCY = - Measure.MeasureDouble.create( + // Measurements + /** + * Latency for a logic operation, which will include latencies for each attempt and exponential + * backoff delays. + */ + static final MeasureLong BIGTABLE_OP_LATENCY = + MeasureLong.create( "cloud.google.com/java/bigtable/op_latency", "Time between request being sent to last row received, " + "or terminal error of the last retry attempt.", MILLISECOND); - static final MeasureDouble BIGTABLE_READ_ROWS_FIRST_ROW_LATENCY = - MeasureDouble.create( + /** + * Number of attempts a logical operation took to complete. Under normal circumstances should be + * 1. + */ + static final MeasureLong BIGTABLE_OP_ATTEMPT_COUNT = + MeasureLong.MeasureLong.create( + "cloud.google.com/java/bigtable/op_attempt_count", + "Number of attempts per operation", + COUNT); + + /** Latency that a single attempt (RPC) took to complete. */ + static final MeasureLong BIGTABLE_ATTEMPT_LATENCY = + MeasureLong.create( + "cloud.google.com/java/bigtable/attempt_latency", + "Duration of an individual operation attempt", + MILLISECOND); + + /** Latency for the caller to see the first row in a ReadRows stream. */ + static final MeasureLong BIGTABLE_READ_ROWS_FIRST_ROW_LATENCY = + MeasureLong.create( "cloud.google.com/java/bigtable/read_rows_first_row_latency", "Time between request being sent to the first row received", MILLISECOND); - - static final MeasureLong BIGTABLE_ROWS_READ_PER_OP = - Measure.MeasureLong.create( - "cloud.google.com/java/bigtable/rows_read_per_op", - "Number of rows received per ReadRows operation", - COUNT); - - static final MeasureLong BIGTABLE_MUTATE_ROWS_ENTRIES_PER_BATCH = - Measure.MeasureLong.create( - "cloud.google.com/java/bigtable/mutations_per_batch", - "Number of mutations per MutateRows request", - COUNT); } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/RpcViewConstants.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/RpcViewConstants.java index 9ee6739006..d21060c4ac 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/RpcViewConstants.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/RpcViewConstants.java @@ -15,11 +15,14 @@ */ package com.google.cloud.bigtable.data.v2.stub.metrics; -import static com.google.cloud.bigtable.data.v2.stub.metrics.RpcMeasureConstants.BIGTABLE_MUTATE_ROWS_ENTRIES_PER_BATCH; +import static com.google.cloud.bigtable.data.v2.stub.metrics.RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID; +import static com.google.cloud.bigtable.data.v2.stub.metrics.RpcMeasureConstants.BIGTABLE_ATTEMPT_LATENCY; +import static com.google.cloud.bigtable.data.v2.stub.metrics.RpcMeasureConstants.BIGTABLE_INSTANCE_ID; import static com.google.cloud.bigtable.data.v2.stub.metrics.RpcMeasureConstants.BIGTABLE_OP; +import static com.google.cloud.bigtable.data.v2.stub.metrics.RpcMeasureConstants.BIGTABLE_OP_ATTEMPT_COUNT; import static com.google.cloud.bigtable.data.v2.stub.metrics.RpcMeasureConstants.BIGTABLE_OP_LATENCY; +import static com.google.cloud.bigtable.data.v2.stub.metrics.RpcMeasureConstants.BIGTABLE_PROJECT_ID; import static com.google.cloud.bigtable.data.v2.stub.metrics.RpcMeasureConstants.BIGTABLE_READ_ROWS_FIRST_ROW_LATENCY; -import static com.google.cloud.bigtable.data.v2.stub.metrics.RpcMeasureConstants.BIGTABLE_ROWS_READ_PER_OP; import static com.google.cloud.bigtable.data.v2.stub.metrics.RpcMeasureConstants.BIGTABLE_STATUS; import com.google.common.collect.ImmutableList; @@ -28,7 +31,6 @@ import io.opencensus.stats.Aggregation.Distribution; import io.opencensus.stats.BucketBoundaries; import io.opencensus.stats.View; -import io.opencensus.tags.TagKey; import java.util.Arrays; class RpcViewConstants { @@ -44,6 +46,13 @@ class RpcViewConstants { 250.0, 300.0, 400.0, 500.0, 650.0, 800.0, 1000.0, 2000.0, 5000.0, 10000.0, 20000.0, 50000.0, 100000.0))); + private static final Aggregation AGGREGATION_ATTEMPT_COUNT = + Distribution.create( + BucketBoundaries.create( + ImmutableList.of( + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, + 100.0))); + private static final Aggregation AGGREGATION_WITH_POWERS_OF_2 = Distribution.create( BucketBoundaries.create( @@ -56,21 +65,31 @@ class RpcViewConstants { * {@link View} for Bigtable client roundtrip latency in milliseconds including all retry * attempts. */ - public static final View BIGTABLE_OP_LATENCY_VIEW = + static final View BIGTABLE_OP_LATENCY_VIEW = View.create( View.Name.create("cloud.google.com/java/bigtable/op_latency"), - "Latency in msecs", + "Operation latency in msecs", BIGTABLE_OP_LATENCY, AGGREGATION_WITH_MILLIS_HISTOGRAM, - ImmutableList.of(BIGTABLE_OP)); + ImmutableList.of( + BIGTABLE_PROJECT_ID, + BIGTABLE_INSTANCE_ID, + BIGTABLE_APP_PROFILE_ID, + BIGTABLE_OP, + BIGTABLE_STATUS)); - static final View BIGTABLE_CLIENT_COMPLETED_OP_VIEW = + static final View BIGTABLE_COMPLETED_OP_VIEW = View.create( View.Name.create("cloud.google.com/java/bigtable/completed_ops"), "Number of completed Bigtable client operations", BIGTABLE_OP_LATENCY, COUNT, - Arrays.asList(BIGTABLE_OP, BIGTABLE_STATUS)); + Arrays.asList( + BIGTABLE_PROJECT_ID, + BIGTABLE_INSTANCE_ID, + BIGTABLE_APP_PROFILE_ID, + BIGTABLE_OP, + BIGTABLE_STATUS)); static final View BIGTABLE_READ_ROWS_FIRST_ROW_LATENCY_VIEW = View.create( @@ -78,21 +97,31 @@ class RpcViewConstants { "Latency to receive the first row in a ReadRows stream", BIGTABLE_READ_ROWS_FIRST_ROW_LATENCY, AGGREGATION_WITH_MILLIS_HISTOGRAM, - ImmutableList.of()); + ImmutableList.of(BIGTABLE_PROJECT_ID, BIGTABLE_INSTANCE_ID, BIGTABLE_APP_PROFILE_ID)); - static final View BIGTABLE_ROWS_READ_PER_OP_VIEW = + static final View BIGTABLE_ATTEMPT_LATENCY_VIEW = View.create( - View.Name.create("cloud.google.com/java/bigtable/rows_per_op"), - "Rows scanned per operation", - BIGTABLE_ROWS_READ_PER_OP, - AGGREGATION_WITH_POWERS_OF_2, - ImmutableList.of()); + View.Name.create("cloud.google.com/java/bigtable/attempt_latency"), + "Attempt latency in msecs", + BIGTABLE_ATTEMPT_LATENCY, + AGGREGATION_WITH_MILLIS_HISTOGRAM, + ImmutableList.of( + BIGTABLE_PROJECT_ID, + BIGTABLE_INSTANCE_ID, + BIGTABLE_APP_PROFILE_ID, + BIGTABLE_OP, + BIGTABLE_STATUS)); - static final View BIGTABLE_MUTATE_ROWS_ENTRIES_PER_BATCH_VIEW = + static final View BIGTABLE_ATTEMPTS_PER_OP_VIEW = View.create( - View.Name.create("cloud.google.com/java/bigtable/mutations_per_batch"), - "Number of mutations sent in a single MutateRowsRequest", - BIGTABLE_MUTATE_ROWS_ENTRIES_PER_BATCH, - AGGREGATION_WITH_POWERS_OF_2, - ImmutableList.of()); + View.Name.create("cloud.google.com/java/bigtable/attempts_per_op"), + "Distribution of attempts per logical operation", + BIGTABLE_OP_ATTEMPT_COUNT, + AGGREGATION_ATTEMPT_COUNT, + ImmutableList.of( + BIGTABLE_PROJECT_ID, + BIGTABLE_INSTANCE_ID, + BIGTABLE_APP_PROFILE_ID, + BIGTABLE_OP, + BIGTABLE_STATUS)); } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/RpcViews.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/RpcViews.java index a8e772e3b1..cc31539496 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/RpcViews.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/RpcViews.java @@ -25,21 +25,15 @@ @BetaApi public class RpcViews { @VisibleForTesting - static final ImmutableSet BIGTABLE_CLIENT_VIEWS_SET = + private static final ImmutableSet BIGTABLE_CLIENT_VIEWS_SET = ImmutableSet.of( RpcViewConstants.BIGTABLE_OP_LATENCY_VIEW, - RpcViewConstants.BIGTABLE_CLIENT_COMPLETED_OP_VIEW, + RpcViewConstants.BIGTABLE_COMPLETED_OP_VIEW, RpcViewConstants.BIGTABLE_READ_ROWS_FIRST_ROW_LATENCY_VIEW, - RpcViewConstants.BIGTABLE_ROWS_READ_PER_OP_VIEW, - RpcViewConstants.BIGTABLE_MUTATE_ROWS_ENTRIES_PER_BATCH_VIEW); + RpcViewConstants.BIGTABLE_ATTEMPT_LATENCY_VIEW, + RpcViewConstants.BIGTABLE_ATTEMPTS_PER_OP_VIEW); - /** - * Registers all Bigtable specific views. - * - *

It is recommended to call this method and {@link - * io.opencensus.contrib.grpc.metrics.RpcViews#registerClientGrpcViews()} before doing any RPC - * call to avoid missing stats. - */ + /** Registers all Bigtable specific views. */ public static void registerBigtableClientViews() { registerBigtableClientViews(Stats.getViewManager()); } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/BulkMutationTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/BulkMutationTest.java index 8b07249735..dc06a58f5c 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/BulkMutationTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/BulkMutationTest.java @@ -147,4 +147,29 @@ public void addRowMutationEntry() { bulkMutation.add(entry); assertThat(bulkMutation.toProto(REQUEST_CONTEXT).getEntriesList()).contains(entry.toProto()); } + + @Test + public void fromProtoTest() { + BulkMutation expected = + BulkMutation.create(TABLE_ID) + .add( + "key", + Mutation.create().setCell("fake-family", "fake-qualifier", 10_000L, "fake-value")); + + MutateRowsRequest protoRequest = expected.toProto(REQUEST_CONTEXT); + BulkMutation actualBulkMutation = BulkMutation.fromProto(protoRequest); + + assertThat(actualBulkMutation.toProto(REQUEST_CONTEXT)).isEqualTo(protoRequest); + + String projectId = "fresh-project"; + String instanceId = "fresh-instance"; + String appProfile = "fresh-app-profile"; + MutateRowsRequest overriddenRequest = + actualBulkMutation.toProto(RequestContext.create(projectId, instanceId, appProfile)); + + assertThat(overriddenRequest).isNotEqualTo(protoRequest); + assertThat(overriddenRequest.getTableName()) + .matches(NameUtil.formatTableName(projectId, instanceId, TABLE_ID)); + assertThat(overriddenRequest.getAppProfileId()).matches(appProfile); + } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutationTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutationTest.java index 044e54f4ef..8a626bb846 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutationTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutationTest.java @@ -162,4 +162,29 @@ public void serializationTest() throws IOException, ClassNotFoundException { ConditionalRowMutation actual = (ConditionalRowMutation) ois.readObject(); assertThat(actual.toProto(REQUEST_CONTEXT)).isEqualTo(expected.toProto(REQUEST_CONTEXT)); } + + @Test + public void fromProtoTest() { + ConditionalRowMutation mutation = + ConditionalRowMutation.create(TABLE_ID, TEST_KEY) + .condition(Filters.FILTERS.key().regex("test")) + .then(Mutation.create().setCell("family1", "qualifier1", 10_000L, "value")) + .otherwise(Mutation.create().deleteFamily("family")); + + CheckAndMutateRowRequest protoRequest = mutation.toProto(REQUEST_CONTEXT); + ConditionalRowMutation actualRequest = ConditionalRowMutation.fromProto(protoRequest); + + assertThat(actualRequest.toProto(REQUEST_CONTEXT)).isEqualTo(protoRequest); + + String projectId = "fresh-project"; + String instanceId = "fresh-instance"; + String appProfile = "fresh-app-profile"; + CheckAndMutateRowRequest overriddenRequest = + actualRequest.toProto(RequestContext.create(projectId, instanceId, appProfile)); + + assertThat(overriddenRequest).isNotEqualTo(protoRequest); + assertThat(overriddenRequest.getTableName()) + .matches(NameUtil.formatTableName(projectId, instanceId, TABLE_ID)); + assertThat(overriddenRequest.getAppProfileId()).matches(appProfile); + } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/MutationTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/MutationTest.java index 2583d51b34..450925ab0a 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/MutationTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/MutationTest.java @@ -254,4 +254,20 @@ public void testWithLongValue() { .setValue(ByteString.copyFrom(Longs.toByteArray(20_000L))) .build()); } + + @Test + public void fromProtoTest() { + mutation + .setCell( + "fake-family", + ByteString.copyFromUtf8("fake-qualifier"), + 1_000, + ByteString.copyFromUtf8("fake-value")) + .deleteCells("fake-family", ByteString.copyFromUtf8("fake-qualifier")) + .deleteFamily("fake-family2"); + + List protoMutation = mutation.getMutations(); + + assertThat(Mutation.fromProto(protoMutation).getMutations()).isEqualTo(protoMutation); + } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRowTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRowTest.java index a318a61f0e..b9e6f9e7ba 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRowTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRowTest.java @@ -113,4 +113,28 @@ public void serializationTest() throws IOException, ClassNotFoundException { ReadModifyWriteRow actual = (ReadModifyWriteRow) ois.readObject(); assertThat(actual.toProto(REQUEST_CONTEXT)).isEqualTo(expected.toProto(REQUEST_CONTEXT)); } + + @Test + public void fromProtoTest() { + ReadModifyWriteRow expected = + ReadModifyWriteRow.create(TABLE_ID, "row-key") + .increment("fake-family", ByteString.copyFromUtf8("fake-qualifier"), 1) + .append("fake-family", "fake-qualifier", "fake-value"); + + ReadModifyWriteRowRequest protoRequest = expected.toProto(REQUEST_CONTEXT); + ReadModifyWriteRow actualRequest = ReadModifyWriteRow.fromProto(protoRequest); + + assertThat(actualRequest.toProto(REQUEST_CONTEXT)).isEqualTo(protoRequest); + + String projectId = "fresh-project"; + String instanceId = "fresh-instance"; + String appProfile = "fresh-app-profile"; + ReadModifyWriteRowRequest overriddenRequest = + actualRequest.toProto(RequestContext.create(projectId, instanceId, appProfile)); + + assertThat(overriddenRequest).isNotEqualTo(protoRequest); + assertThat(overriddenRequest.getTableName()) + .matches(NameUtil.formatTableName(projectId, instanceId, TABLE_ID)); + assertThat(overriddenRequest.getAppProfileId()).matches(appProfile); + } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/RowMutationTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/RowMutationTest.java index 553d7ce9d5..b401ad5ef3 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/RowMutationTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/RowMutationTest.java @@ -139,4 +139,28 @@ public void testWithLongValue() { .setValue(ByteString.copyFrom(Longs.toByteArray(100_000L))) .build()); } + + @Test + public void fromProtoTest() { + RowMutation rowMutation = + RowMutation.create("fake-table", "fake-key") + .setCell("fake-family", "fake-qualifier-1", "fake-value") + .setCell("fake-family", "fake-qualifier-2", 30_000L, "fake-value-2"); + + MutateRowRequest protoRequest = rowMutation.toProto(REQUEST_CONTEXT); + RowMutation actualRequest = RowMutation.fromProto(protoRequest); + + assertThat(actualRequest.toProto(REQUEST_CONTEXT)).isEqualTo(protoRequest); + + String projectId = "fresh-project"; + String instanceId = "fresh-instance"; + String appProfile = "fresh-app-profile"; + MutateRowRequest overriddenRequest = + actualRequest.toProto(RequestContext.create(projectId, instanceId, appProfile)); + + assertThat(overriddenRequest).isNotEqualTo(protoRequest); + assertThat(overriddenRequest.getTableName()) + .matches(NameUtil.formatTableName(projectId, instanceId, TABLE_ID)); + assertThat(overriddenRequest.getAppProfileId()).matches(appProfile); + } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubTest.java new file mode 100644 index 0000000000..be2d9c2a0f --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubTest.java @@ -0,0 +1,146 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.stub; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.gax.core.NoCredentialsProvider; +import com.google.api.gax.grpc.testing.InProcessServer; +import com.google.api.gax.grpc.testing.LocalChannelProvider; +import com.google.api.gax.rpc.ServerStreamingCallable; +import com.google.bigtable.v2.BigtableGrpc; +import com.google.bigtable.v2.ReadRowsRequest; +import com.google.bigtable.v2.ReadRowsResponse; +import com.google.bigtable.v2.RowSet; +import com.google.cloud.bigtable.admin.v2.internal.NameUtil; +import com.google.cloud.bigtable.data.v2.internal.RequestContext; +import com.google.cloud.bigtable.data.v2.models.DefaultRowAdapter; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.common.collect.Queues; +import com.google.protobuf.ByteString; +import com.google.protobuf.BytesValue; +import com.google.protobuf.StringValue; +import io.grpc.stub.StreamObserver; +import java.io.IOException; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class EnhancedBigtableStubTest { + + private static final String PROJECT_ID = "fake-project"; + private static final String INSTANCE_ID = "fake-instance"; + private static final String FAKE_HOST_NAME = "fake-stub-host:123"; + private static final String TABLE_NAME = + NameUtil.formatTableName(PROJECT_ID, INSTANCE_ID, "fake-table"); + private static final String APP_PROFILE_ID = "app-profile-id"; + + private InProcessServer server; + private FakeDataService fakeDataService; + private EnhancedBigtableStub enhancedBigtableStub; + + @Before + public void setUp() throws IOException, IllegalAccessException, InstantiationException { + fakeDataService = new FakeDataService(); + server = new InProcessServer<>(fakeDataService, FAKE_HOST_NAME); + server.start(); + + EnhancedBigtableStubSettings enhancedBigtableStubSettings = + EnhancedBigtableStubSettings.newBuilder() + .setProjectId(PROJECT_ID) + .setInstanceId(INSTANCE_ID) + .setAppProfileId(APP_PROFILE_ID) + .setCredentialsProvider(NoCredentialsProvider.create()) + .setEndpoint(FAKE_HOST_NAME) + .setTransportChannelProvider(LocalChannelProvider.create(FAKE_HOST_NAME)) + .build(); + + enhancedBigtableStub = EnhancedBigtableStub.create(enhancedBigtableStubSettings); + } + + @After + public void tearDown() { + server.stop(); + } + + @Test + public void testCreateReadRowsCallable() throws InterruptedException { + ServerStreamingCallable streamingCallable = + enhancedBigtableStub.createReadRowsCallable(new DefaultRowAdapter()); + + Query request = Query.create("table-id").rowKey("row-key"); + streamingCallable.call(request).iterator().next(); + ReadRowsRequest expected = + request.toProto(RequestContext.create(PROJECT_ID, INSTANCE_ID, APP_PROFILE_ID)); + assertThat(fakeDataService.popLastRequest()).isEqualTo(expected); + } + + @Test + public void testCreateReadRowsRawCallable() throws InterruptedException { + ServerStreamingCallable callable = + enhancedBigtableStub.createReadRowsRawCallable(new DefaultRowAdapter()); + + ReadRowsRequest expectedRequest = + ReadRowsRequest.newBuilder() + .setTableName(TABLE_NAME) + .setAppProfileId("app-profile-1") + .setRows(RowSet.newBuilder().addRowKeys(ByteString.copyFromUtf8("test-row-key"))) + .build(); + callable.call(expectedRequest).iterator().next(); + assertThat(fakeDataService.popLastRequest()).isEqualTo(expectedRequest); + + ReadRowsRequest expectedRequest2 = + ReadRowsRequest.newBuilder() + .setTableName(TABLE_NAME) + .setAppProfileId("app-profile-2") + .build(); + callable.call(expectedRequest2).iterator().next(); + assertThat(fakeDataService.popLastRequest()).isEqualTo(expectedRequest2); + } + + private static class FakeDataService extends BigtableGrpc.BigtableImplBase { + final BlockingQueue requests = Queues.newLinkedBlockingDeque(); + + @SuppressWarnings("unchecked") + T popLastRequest() throws InterruptedException { + return (T) requests.poll(1, TimeUnit.SECONDS); + } + + @Override + public void readRows( + ReadRowsRequest request, StreamObserver responseObserver) { + requests.add(request); + // Dummy row for stream + responseObserver.onNext( + ReadRowsResponse.newBuilder() + .addChunks( + ReadRowsResponse.CellChunk.newBuilder() + .setCommitRow(true) + .setRowKey(ByteString.copyFromUtf8("a")) + .setFamilyName(StringValue.getDefaultInstance()) + .setQualifier(BytesValue.getDefaultInstance()) + .setValueSize(0)) + .build()); + responseObserver.onCompleted(); + } + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/HeadersTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/HeadersTest.java index c6c5740d8d..6bd763cda0 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/HeadersTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/HeadersTest.java @@ -175,7 +175,7 @@ private void verifyHeaderSent() { String requestParamsvalue = metadata.get(X_GOOG_REQUEST_PARAMS_KEY); assertThat(requestParamsvalue).containsMatch("(^|.*&)table_name=" + TABLE_NAME + "($|&.*)"); assertThat(requestParamsvalue) - .containsMatch("(^|.*&)app_profile_id=" + APP_PROFILE_ID + "($|&.*)"); + .containsMatch("(^|.*&)app_profile=" + APP_PROFILE_ID + "($|&.*)"); String apiClientValue = metadata.get(API_CLIENT_HEADER_KEY); assertThat(apiClientValue).containsMatch("gl-java/[.\\d_]+"); diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/CompositeTracerTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/CompositeTracerTest.java new file mode 100644 index 0000000000..cedb227bad --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/CompositeTracerTest.java @@ -0,0 +1,174 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.stub.metrics; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.api.gax.tracing.ApiTracer; +import com.google.api.gax.tracing.ApiTracer.Scope; +import com.google.common.collect.ImmutableList; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.threeten.bp.Duration; + +@RunWith(JUnit4.class) +public class CompositeTracerTest { + @Rule public final MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Mock private ApiTracer child1; + @Mock private ApiTracer child2; + + private CompositeTracer compositeTracer; + + @Before + public void setup() { + compositeTracer = new CompositeTracer(ImmutableList.of(child1, child2)); + } + + @Test + public void testInScope() { + Scope scope1 = mock(Scope.class); + when(child1.inScope()).thenReturn(scope1); + + Scope scope2 = mock(Scope.class); + when(child2.inScope()).thenReturn(scope2); + + Scope parentScope = compositeTracer.inScope(); + + parentScope.close(); + verify(scope1, times(1)).close(); + } + + @Test + public void testOperationSucceeded() { + compositeTracer.operationSucceeded(); + verify(child1, times(1)).operationSucceeded(); + verify(child2, times(1)).operationSucceeded(); + } + + @Test + public void testOperationCancelled() { + compositeTracer.operationCancelled(); + verify(child1, times(1)).operationCancelled(); + verify(child2, times(1)).operationCancelled(); + } + + @Test + public void testOperationFailed() { + RuntimeException error = new RuntimeException(); + compositeTracer.operationFailed(error); + verify(child1, times(1)).operationFailed(error); + verify(child2, times(1)).operationFailed(error); + } + + @Test + public void testConnectionSelected() { + compositeTracer.connectionSelected("connection-one"); + verify(child1, times(1)).connectionSelected("connection-one"); + verify(child2, times(1)).connectionSelected("connection-one"); + } + + @Test + public void testAttemptStarted() { + compositeTracer.attemptStarted(3); + verify(child1, times(1)).attemptStarted(3); + verify(child2, times(1)).attemptStarted(3); + } + + @Test + public void testAttemptSucceeded() { + compositeTracer.attemptSucceeded(); + verify(child1, times(1)).attemptSucceeded(); + verify(child2, times(1)).attemptSucceeded(); + } + + @Test + public void testAttemptCancelled() { + compositeTracer.attemptCancelled(); + verify(child1, times(1)).attemptCancelled(); + verify(child2, times(1)).attemptCancelled(); + } + + @Test + public void testAttemptFailed() { + RuntimeException error = new RuntimeException(); + Duration delay = Duration.ofMillis(10); + compositeTracer.attemptFailed(error, delay); + verify(child1, times(1)).attemptFailed(error, delay); + verify(child2, times(1)).attemptFailed(error, delay); + } + + @Test + public void testAttemptFailedRetriesExhausted() { + RuntimeException error = new RuntimeException(); + compositeTracer.attemptFailedRetriesExhausted(error); + verify(child1, times(1)).attemptFailedRetriesExhausted(error); + verify(child2, times(1)).attemptFailedRetriesExhausted(error); + } + + @Test + public void testAttemptPermanentFailure() { + RuntimeException error = new RuntimeException(); + compositeTracer.attemptPermanentFailure(error); + verify(child1, times(1)).attemptPermanentFailure(error); + verify(child2, times(1)).attemptPermanentFailure(error); + } + + @Test + public void testLroStartFailed() { + RuntimeException error = new RuntimeException(); + compositeTracer.lroStartFailed(error); + verify(child1, times(1)).lroStartFailed(error); + verify(child2, times(1)).lroStartFailed(error); + } + + @Test + public void testLroStartSucceeded() { + compositeTracer.lroStartSucceeded(); + verify(child1, times(1)).lroStartSucceeded(); + verify(child2, times(1)).lroStartSucceeded(); + } + + @Test + public void testResponseReceived() { + compositeTracer.responseReceived(); + verify(child1, times(1)).responseReceived(); + verify(child2, times(1)).responseReceived(); + } + + @Test + public void testRequestSent() { + compositeTracer.requestSent(); + verify(child1, times(1)).requestSent(); + verify(child2, times(1)).requestSent(); + } + + @Test + public void testBatchRequestSent() { + compositeTracer.batchRequestSent(2, 20); + verify(child1, times(1)).batchRequestSent(2, 20); + verify(child2, times(1)).batchRequestSent(2, 20); + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MeasureMutateRowsCallableTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MeasureMutateRowsCallableTest.java deleted file mode 100644 index 2b47282da3..0000000000 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MeasureMutateRowsCallableTest.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.cloud.bigtable.data.v2.stub.metrics; - -import static com.google.common.truth.Truth.assertThat; - -import com.google.api.core.ApiFuture; -import com.google.api.core.ApiFutures; -import com.google.api.gax.core.FakeApiClock; -import com.google.api.gax.grpc.GrpcStatusCode; -import com.google.api.gax.rpc.ApiCallContext; -import com.google.api.gax.rpc.DeadlineExceededException; -import com.google.api.gax.rpc.UnaryCallable; -import com.google.cloud.bigtable.data.v2.models.BulkMutation; -import com.google.cloud.bigtable.data.v2.models.Mutation; -import com.google.cloud.bigtable.data.v2.stub.metrics.StatsTestUtils.FakeStatsRecorder; -import com.google.cloud.bigtable.data.v2.stub.metrics.StatsTestUtils.FakeTagger; -import com.google.cloud.bigtable.data.v2.stub.metrics.StatsTestUtils.MetricsRecord; -import io.grpc.Status.Code; -import io.opencensus.tags.TagValue; -import java.util.concurrent.TimeUnit; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; -import org.mockito.stubbing.Answer; - -@RunWith(JUnit4.class) -public class MeasureMutateRowsCallableTest { - private static final String METHOD_NAME = "Bigtable.MutateRows"; - @Rule public final MockitoRule rule = MockitoJUnit.rule(); - - private FakeTagger tagger; - - private FakeStatsRecorder statsRecorder; - - private FakeApiClock clock; - - @Mock private UnaryCallable innerCallable; - - private MeasuredMutateRowsCallable callable; - - @Before - public void setUp() { - tagger = new FakeTagger(); - statsRecorder = new FakeStatsRecorder(); - clock = new FakeApiClock(0); - - callable = - new MeasuredMutateRowsCallable(innerCallable, METHOD_NAME, tagger, statsRecorder, clock); - } - - @Test - public void testOk() { - Mockito.when( - innerCallable.futureCall( - Mockito.any(BulkMutation.class), Mockito.any(ApiCallContext.class))) - .thenAnswer( - new Answer>() { - @Override - public ApiFuture answer(InvocationOnMock invocationOnMock) { - clock.incrementNanoTime(TimeUnit.MILLISECONDS.toNanos(3)); - return ApiFutures.immediateFuture(null); - } - }); - - callable.call( - BulkMutation.create("tableID") - .add("rowKey", Mutation.create()) - .add("rowKey2", Mutation.create())); - - MetricsRecord metricsRecord = statsRecorder.pollRecord(); - - assertThat(metricsRecord.metrics).containsEntry(RpcMeasureConstants.BIGTABLE_OP_LATENCY, 3.0); - assertThat(metricsRecord.metrics) - .containsEntry(RpcMeasureConstants.BIGTABLE_MUTATE_ROWS_ENTRIES_PER_BATCH, 2L); - assertThat(metricsRecord.tags) - .containsEntry(RpcMeasureConstants.BIGTABLE_OP, TagValue.create(METHOD_NAME)); - assertThat(metricsRecord.tags) - .containsEntry(RpcMeasureConstants.BIGTABLE_STATUS, TagValue.create("OK")); - } - - @Test - public void testFailure() { - Mockito.when( - innerCallable.futureCall( - Mockito.any(BulkMutation.class), Mockito.any(ApiCallContext.class))) - .thenAnswer( - new Answer>() { - @Override - public ApiFuture answer(InvocationOnMock invocationOnMock) { - clock.incrementNanoTime(TimeUnit.MILLISECONDS.toNanos(3)); - return ApiFutures.immediateFailedFuture( - new DeadlineExceededException( - "timeout!", null, GrpcStatusCode.of(Code.DEADLINE_EXCEEDED), true)); - } - }); - - Throwable actualError = null; - - try { - callable.call( - BulkMutation.create("tableID") - .add("rowKey", Mutation.create()) - .add("rowKey2", Mutation.create())); - } catch (Throwable e) { - actualError = e; - } - - assertThat(actualError).isInstanceOf(DeadlineExceededException.class); - - MetricsRecord metricsRecord = statsRecorder.pollRecord(); - - assertThat(metricsRecord.metrics).containsEntry(RpcMeasureConstants.BIGTABLE_OP_LATENCY, 3.0); - assertThat(metricsRecord.metrics) - .containsEntry(RpcMeasureConstants.BIGTABLE_MUTATE_ROWS_ENTRIES_PER_BATCH, 2L); - assertThat(metricsRecord.tags) - .containsEntry(RpcMeasureConstants.BIGTABLE_OP, TagValue.create(METHOD_NAME)); - assertThat(metricsRecord.tags) - .containsEntry(RpcMeasureConstants.BIGTABLE_STATUS, TagValue.create("DEADLINE_EXCEEDED")); - } -} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MeasuredReadRowsCallableTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MeasuredReadRowsCallableTest.java deleted file mode 100644 index 2c08638959..0000000000 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MeasuredReadRowsCallableTest.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.cloud.bigtable.data.v2.stub.metrics; - -import static com.google.common.truth.Truth.assertThat; - -import com.google.api.gax.core.FakeApiClock; -import com.google.api.gax.grpc.GrpcStatusCode; -import com.google.api.gax.rpc.DeadlineExceededException; -import com.google.cloud.bigtable.data.v2.models.Query; -import com.google.cloud.bigtable.data.v2.stub.metrics.StatsTestUtils.FakeStatsRecorder; -import com.google.cloud.bigtable.data.v2.stub.metrics.StatsTestUtils.FakeTagger; -import com.google.cloud.bigtable.data.v2.stub.metrics.StatsTestUtils.MetricsRecord; -import com.google.cloud.bigtable.gaxx.testing.MockStreamingApi.MockServerStreamingCall; -import com.google.cloud.bigtable.gaxx.testing.MockStreamingApi.MockServerStreamingCallable; -import io.grpc.Status.Code; -import io.opencensus.tags.TagValue; -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; - -@RunWith(JUnit4.class) -public class MeasuredReadRowsCallableTest { - @Rule public final MockitoRule rule = MockitoJUnit.rule(); - - private FakeTagger tagger; - - private FakeStatsRecorder statsRecorder; - - private FakeApiClock clock; - - private MockServerStreamingCallable innerCallable; - - private MeasuredReadRowsCallable callable; - - @Before - public void setUp() { - innerCallable = new MockServerStreamingCallable<>(); - - tagger = new FakeTagger(); - statsRecorder = new FakeStatsRecorder(); - clock = new FakeApiClock(0); - - callable = - new MeasuredReadRowsCallable<>( - innerCallable, "Bigtable.ReadRows", tagger, statsRecorder, clock); - } - - @Test - public void testOk() { - new Thread() { - @Override - public void run() { - MockServerStreamingCall lastCall = null; - - for (int i = 0; i < 10 && lastCall == null; i++) { - lastCall = innerCallable.popLastCall(); - } - clock.incrementNanoTime(TimeUnit.MILLISECONDS.toNanos(2)); - lastCall.getController().popLastPull(); - lastCall.getController().getObserver().onResponse("row0"); - - clock.incrementNanoTime(TimeUnit.MILLISECONDS.toNanos(3)); - lastCall.getController().popLastPull(); - lastCall.getController().getObserver().onResponse("row1"); - lastCall.getController().getObserver().onComplete(); - } - }.start(); - - List results = callable.all().call(Query.create("fake-table")); - - assertThat(results).containsExactly("row0", "row1"); - - MetricsRecord metricsRecord = statsRecorder.pollRecord(); - - assertThat(metricsRecord.metrics).containsEntry(RpcMeasureConstants.BIGTABLE_OP_LATENCY, 5.0); - assertThat(metricsRecord.metrics) - .containsEntry(RpcMeasureConstants.BIGTABLE_READ_ROWS_FIRST_ROW_LATENCY, 2.0); - assertThat(metricsRecord.metrics) - .containsEntry(RpcMeasureConstants.BIGTABLE_ROWS_READ_PER_OP, 2L); - assertThat(metricsRecord.tags) - .containsEntry(RpcMeasureConstants.BIGTABLE_OP, TagValue.create("Bigtable.ReadRows")); - assertThat(metricsRecord.tags) - .containsEntry(RpcMeasureConstants.BIGTABLE_STATUS, TagValue.create("OK")); - } - - @Test - public void testEmpty() { - new Thread() { - @Override - public void run() { - MockServerStreamingCall lastCall = null; - - for (int i = 0; i < 10 && lastCall == null; i++) { - lastCall = innerCallable.popLastCall(); - } - clock.incrementNanoTime(TimeUnit.MILLISECONDS.toNanos(2)); - lastCall.getController().getObserver().onComplete(); - } - }.start(); - - List results = callable.all().call(Query.create("fake-table")); - - assertThat(results).isEmpty(); - - MetricsRecord metricsRecord = statsRecorder.pollRecord(); - - assertThat(metricsRecord.metrics).containsEntry(RpcMeasureConstants.BIGTABLE_OP_LATENCY, 2.0); - assertThat(metricsRecord.metrics) - .doesNotContainKey(RpcMeasureConstants.BIGTABLE_READ_ROWS_FIRST_ROW_LATENCY); - assertThat(metricsRecord.metrics) - .containsEntry(RpcMeasureConstants.BIGTABLE_ROWS_READ_PER_OP, 0L); - assertThat(metricsRecord.tags) - .containsEntry(RpcMeasureConstants.BIGTABLE_OP, TagValue.create("Bigtable.ReadRows")); - assertThat(metricsRecord.tags) - .containsEntry(RpcMeasureConstants.BIGTABLE_STATUS, TagValue.create("OK")); - } - - @Test - public void testFailure() { - new Thread() { - @Override - public void run() { - MockServerStreamingCall lastCall = null; - - for (int i = 0; i < 10 && lastCall == null; i++) { - lastCall = innerCallable.popLastCall(); - } - clock.incrementNanoTime(TimeUnit.MILLISECONDS.toNanos(2)); - lastCall - .getController() - .getObserver() - .onError( - new DeadlineExceededException( - "timeout!", null, GrpcStatusCode.of(Code.DEADLINE_EXCEEDED), true)); - } - }.start(); - - Throwable actualError = null; - try { - callable.all().call(Query.create("fake-table")); - } catch (Throwable e) { - actualError = e; - } - - assertThat(actualError).isInstanceOf(DeadlineExceededException.class); - - MetricsRecord metricsRecord = statsRecorder.pollRecord(); - - assertThat(metricsRecord.metrics).containsEntry(RpcMeasureConstants.BIGTABLE_OP_LATENCY, 2.0); - assertThat(metricsRecord.metrics) - .doesNotContainKey(RpcMeasureConstants.BIGTABLE_READ_ROWS_FIRST_ROW_LATENCY); - assertThat(metricsRecord.metrics) - .containsEntry(RpcMeasureConstants.BIGTABLE_ROWS_READ_PER_OP, 0L); - assertThat(metricsRecord.tags) - .containsEntry(RpcMeasureConstants.BIGTABLE_OP, TagValue.create("Bigtable.ReadRows")); - assertThat(metricsRecord.tags) - .containsEntry(RpcMeasureConstants.BIGTABLE_STATUS, TagValue.create("DEADLINE_EXCEEDED")); - } - - @Test - public void testFailureAfterData() { - new Thread() { - @Override - public void run() { - MockServerStreamingCall lastCall = null; - - for (int i = 0; i < 10 && lastCall == null; i++) { - lastCall = innerCallable.popLastCall(); - } - clock.incrementNanoTime(TimeUnit.MILLISECONDS.toNanos(2)); - lastCall.getController().popLastPull(); - lastCall.getController().getObserver().onResponse("row0"); - - clock.incrementNanoTime(TimeUnit.MILLISECONDS.toNanos(3)); - lastCall - .getController() - .getObserver() - .onError( - new DeadlineExceededException( - "timeout!", null, GrpcStatusCode.of(Code.DEADLINE_EXCEEDED), true)); - } - }.start(); - - Throwable actualError = null; - try { - callable.all().call(Query.create("fake-table")); - } catch (Throwable e) { - actualError = e; - } - - assertThat(actualError).isInstanceOf(DeadlineExceededException.class); - - MetricsRecord metricsRecord = statsRecorder.pollRecord(); - - assertThat(metricsRecord.metrics).containsEntry(RpcMeasureConstants.BIGTABLE_OP_LATENCY, 5.0); - assertThat(metricsRecord.metrics) - .containsEntry(RpcMeasureConstants.BIGTABLE_READ_ROWS_FIRST_ROW_LATENCY, 2.0); - assertThat(metricsRecord.metrics) - .containsEntry(RpcMeasureConstants.BIGTABLE_ROWS_READ_PER_OP, 1L); - assertThat(metricsRecord.tags) - .containsEntry(RpcMeasureConstants.BIGTABLE_OP, TagValue.create("Bigtable.ReadRows")); - assertThat(metricsRecord.tags) - .containsEntry(RpcMeasureConstants.BIGTABLE_STATUS, TagValue.create("DEADLINE_EXCEEDED")); - } -} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MeasuredUnaryCallableTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MeasuredUnaryCallableTest.java deleted file mode 100644 index 41006148fa..0000000000 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MeasuredUnaryCallableTest.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.cloud.bigtable.data.v2.stub.metrics; - -import static com.google.common.truth.Truth.assertThat; - -import com.google.api.core.ApiFuture; -import com.google.api.core.ApiFutures; -import com.google.api.gax.core.FakeApiClock; -import com.google.api.gax.grpc.GrpcStatusCode; -import com.google.api.gax.rpc.ApiCallContext; -import com.google.api.gax.rpc.DeadlineExceededException; -import com.google.api.gax.rpc.UnaryCallable; -import com.google.cloud.bigtable.data.v2.stub.metrics.StatsTestUtils.FakeStatsRecorder; -import com.google.cloud.bigtable.data.v2.stub.metrics.StatsTestUtils.FakeTagger; -import com.google.cloud.bigtable.data.v2.stub.metrics.StatsTestUtils.MetricsRecord; -import io.grpc.Status.Code; -import io.opencensus.tags.TagValue; -import java.util.concurrent.TimeUnit; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; -import org.mockito.stubbing.Answer; - -@RunWith(JUnit4.class) -public class MeasuredUnaryCallableTest { - private static final String FAKE_METHOD = "Bigtable.FakeMethod"; - - @Rule public final MockitoRule rule = MockitoJUnit.rule(); - - private FakeTagger tagger; - - private FakeStatsRecorder statsRecorder; - - private FakeApiClock clock; - - @Mock private UnaryCallable innerCallable; - - private MeasuredUnaryCallable callable; - - @Before - public void setUp() { - tagger = new FakeTagger(); - statsRecorder = new FakeStatsRecorder(); - clock = new FakeApiClock(0); - - callable = - new MeasuredUnaryCallable<>(innerCallable, FAKE_METHOD, tagger, statsRecorder, clock); - } - - @Test - public void testOk() { - Mockito.when(innerCallable.futureCall(Mockito.anyString(), Mockito.any(ApiCallContext.class))) - .thenAnswer( - new Answer>() { - @Override - public ApiFuture answer(InvocationOnMock invocationOnMock) { - clock.incrementNanoTime(TimeUnit.MILLISECONDS.toNanos(2)); - return ApiFutures.immediateFuture("response"); - } - }); - - String response = callable.call("request"); - - assertThat(response).isEqualTo("response"); - - MetricsRecord metricsRecord = statsRecorder.pollRecord(); - - assertThat(metricsRecord.metrics).containsEntry(RpcMeasureConstants.BIGTABLE_OP_LATENCY, 2.0); - assertThat(metricsRecord.tags) - .containsEntry(RpcMeasureConstants.BIGTABLE_OP, TagValue.create(FAKE_METHOD)); - assertThat(metricsRecord.tags) - .containsEntry(RpcMeasureConstants.BIGTABLE_STATUS, TagValue.create("OK")); - } - - @Test - public void testFailure() { - Mockito.when(innerCallable.futureCall(Mockito.anyString(), Mockito.any(ApiCallContext.class))) - .thenAnswer( - new Answer>() { - @Override - public ApiFuture answer(InvocationOnMock invocationOnMock) { - clock.incrementNanoTime(TimeUnit.MILLISECONDS.toNanos(2)); - return ApiFutures.immediateFailedFuture( - new DeadlineExceededException( - "timeout!", null, GrpcStatusCode.of(Code.DEADLINE_EXCEEDED), true)); - } - }); - - Throwable actualError = null; - try { - callable.call("request"); - } catch (Throwable e) { - actualError = e; - } - - assertThat(actualError).isInstanceOf(DeadlineExceededException.class); - - MetricsRecord metricsRecord = statsRecorder.pollRecord(); - - assertThat(metricsRecord.metrics).containsEntry(RpcMeasureConstants.BIGTABLE_OP_LATENCY, 2.0); - assertThat(metricsRecord.tags) - .containsEntry(RpcMeasureConstants.BIGTABLE_OP, TagValue.create(FAKE_METHOD)); - assertThat(metricsRecord.tags) - .containsEntry(RpcMeasureConstants.BIGTABLE_STATUS, TagValue.create("DEADLINE_EXCEEDED")); - } -} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracerTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracerTest.java new file mode 100644 index 0000000000..9fd78c5cd5 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracerTest.java @@ -0,0 +1,404 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.stub.metrics; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doAnswer; + +import com.google.api.gax.rpc.ClientContext; +import com.google.bigtable.v2.BigtableGrpc; +import com.google.bigtable.v2.ReadRowsRequest; +import com.google.bigtable.v2.ReadRowsResponse; +import com.google.bigtable.v2.ReadRowsResponse.CellChunk; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStub; +import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStubSettings; +import com.google.common.base.Stopwatch; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Range; +import com.google.protobuf.ByteString; +import com.google.protobuf.BytesValue; +import com.google.protobuf.StringValue; +import io.grpc.Server; +import io.grpc.ServerBuilder; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.StreamObserver; +import io.opencensus.common.Function; +import io.opencensus.impl.stats.StatsComponentImpl; +import io.opencensus.stats.AggregationData; +import io.opencensus.stats.AggregationData.CountData; +import io.opencensus.stats.AggregationData.DistributionData; +import io.opencensus.stats.AggregationData.LastValueDataDouble; +import io.opencensus.stats.AggregationData.LastValueDataLong; +import io.opencensus.stats.AggregationData.SumDataDouble; +import io.opencensus.stats.AggregationData.SumDataLong; +import io.opencensus.stats.View; +import io.opencensus.stats.ViewData; +import io.opencensus.tags.TagKey; +import io.opencensus.tags.TagValue; +import io.opencensus.tags.Tags; +import java.net.ServerSocket; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.mockito.stubbing.Answer; + +@RunWith(JUnit4.class) +public class MetricsTracerTest { + private static final String PROJECT_ID = "fake-project"; + private static final String INSTANCE_ID = "fake-instance"; + private static final String APP_PROFILE_ID = "default"; + private static final String TABLE_ID = "fake-table"; + + private static final ReadRowsResponse DEFAULT_READ_ROWS_RESPONSES = + ReadRowsResponse.newBuilder() + .addChunks( + CellChunk.newBuilder() + .setRowKey(ByteString.copyFromUtf8("fake-key")) + .setFamilyName(StringValue.of("cf")) + .setQualifier(BytesValue.newBuilder().setValue(ByteString.copyFromUtf8("q"))) + .setTimestampMicros(1_000) + .setValue(ByteString.copyFromUtf8("value")) + .setCommitRow(true)) + .build(); + + @Rule public final MockitoRule mockitoRule = MockitoJUnit.rule(); + + private Server server; + + @Mock(answer = Answers.CALLS_REAL_METHODS) + private BigtableGrpc.BigtableImplBase mockService; + + private StatsComponentImpl localStats = new StatsComponentImpl(); + private EnhancedBigtableStub stub; + + @Before + public void setUp() throws Exception { + int port; + try (ServerSocket ss = new ServerSocket(0)) { + port = ss.getLocalPort(); + } + server = ServerBuilder.forPort(port).addService(mockService).build(); + server.start(); + + RpcViews.registerBigtableClientViews(localStats.getViewManager()); + + BigtableDataSettings settings = + BigtableDataSettings.newBuilderForEmulator(port) + .setProjectId(PROJECT_ID) + .setInstanceId(INSTANCE_ID) + .setAppProfileId(APP_PROFILE_ID) + .build(); + EnhancedBigtableStubSettings stubSettings = settings.getStubSettings(); + + stub = + new EnhancedBigtableStub( + stubSettings, + ClientContext.create(stubSettings), + Tags.getTagger(), + localStats.getStatsRecorder()); + } + + @After + public void tearDown() { + stub.close(); + server.shutdown(); + } + + @Test + public void testReadRowsLatency() throws InterruptedException { + final long sleepTime = 50; + + doAnswer( + new Answer() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + @SuppressWarnings("unchecked") + StreamObserver observer = + (StreamObserver) invocation.getArguments()[1]; + Thread.sleep(sleepTime); + observer.onNext(DEFAULT_READ_ROWS_RESPONSES); + observer.onCompleted(); + return null; + } + }) + .when(mockService) + .readRows(any(ReadRowsRequest.class), anyObserver(ReadRowsResponse.class)); + + Stopwatch stopwatch = Stopwatch.createStarted(); + Lists.newArrayList(stub.readRowsCallable().call(Query.create(TABLE_ID))); + long elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS); + + // Give OpenCensus a chance to update the views asynchronously. + Thread.sleep(100); + + long opLatency = + getAggregationValueAsLong( + RpcViewConstants.BIGTABLE_OP_LATENCY_VIEW, + ImmutableMap.of( + RpcMeasureConstants.BIGTABLE_OP, TagValue.create("Bigtable.ReadRows"), + RpcMeasureConstants.BIGTABLE_STATUS, TagValue.create("OK"))); + assertThat(opLatency).isIn(Range.closed(sleepTime, elapsed)); + } + + @Test + public void testReadRowsOpCount() throws InterruptedException { + doAnswer( + new Answer() { + @Override + public Object answer(InvocationOnMock invocation) { + @SuppressWarnings("unchecked") + StreamObserver observer = + (StreamObserver) invocation.getArguments()[1]; + observer.onNext(DEFAULT_READ_ROWS_RESPONSES); + observer.onCompleted(); + return null; + } + }) + .when(mockService) + .readRows(any(ReadRowsRequest.class), anyObserver(ReadRowsResponse.class)); + + Lists.newArrayList(stub.readRowsCallable().call(Query.create(TABLE_ID))); + Lists.newArrayList(stub.readRowsCallable().call(Query.create(TABLE_ID))); + + // Give OpenCensus a chance to update the views asynchronously. + Thread.sleep(100); + + long opLatency = + getAggregationValueAsLong( + RpcViewConstants.BIGTABLE_COMPLETED_OP_VIEW, + ImmutableMap.of( + RpcMeasureConstants.BIGTABLE_OP, TagValue.create("Bigtable.ReadRows"), + RpcMeasureConstants.BIGTABLE_STATUS, TagValue.create("OK"))); + assertThat(opLatency).isEqualTo(2); + } + + @Test + public void testReadRowsFirstRow() throws InterruptedException { + final long beforeSleep = 50; + final long afterSleep = 50; + + doAnswer( + new Answer() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + @SuppressWarnings("unchecked") + StreamObserver observer = + (StreamObserver) invocation.getArguments()[1]; + Thread.sleep(beforeSleep); + observer.onNext(DEFAULT_READ_ROWS_RESPONSES); + Thread.sleep(afterSleep); + observer.onCompleted(); + return null; + } + }) + .when(mockService) + .readRows(any(ReadRowsRequest.class), anyObserver(ReadRowsResponse.class)); + + Stopwatch stopwatch = Stopwatch.createStarted(); + Lists.newArrayList(stub.readRowsCallable().call(Query.create(TABLE_ID))); + long elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS); + + // Give OpenCensus a chance to update the views asynchronously. + Thread.sleep(100); + + long firstRowLatency = + getAggregationValueAsLong( + RpcViewConstants.BIGTABLE_READ_ROWS_FIRST_ROW_LATENCY_VIEW, + ImmutableMap.of()); + + // adding buffer time to the upper range to allow for a race between the emulator and the client + // recording the duration + assertThat(firstRowLatency).isIn(Range.closed(beforeSleep, elapsed - afterSleep / 2)); + } + + @Test + public void testReadRowsAttemptsPerOp() throws InterruptedException { + final AtomicInteger callCount = new AtomicInteger(0); + + doAnswer( + new Answer() { + @Override + public Object answer(InvocationOnMock invocation) { + @SuppressWarnings("unchecked") + StreamObserver observer = + (StreamObserver) invocation.getArguments()[1]; + + // First call will trigger a transient error + if (callCount.getAndIncrement() == 0) { + observer.onError(new StatusRuntimeException(Status.UNAVAILABLE)); + return null; + } + + // Next attempt will return a row + observer.onNext(DEFAULT_READ_ROWS_RESPONSES); + observer.onCompleted(); + return null; + } + }) + .when(mockService) + .readRows(any(ReadRowsRequest.class), anyObserver(ReadRowsResponse.class)); + + Lists.newArrayList(stub.readRowsCallable().call(Query.create(TABLE_ID))); + + // Give OpenCensus a chance to update the views asynchronously. + Thread.sleep(100); + + long opLatency = + getAggregationValueAsLong( + RpcViewConstants.BIGTABLE_ATTEMPTS_PER_OP_VIEW, + ImmutableMap.of( + RpcMeasureConstants.BIGTABLE_OP, TagValue.create("Bigtable.ReadRows"), + RpcMeasureConstants.BIGTABLE_STATUS, TagValue.create("OK"))); + assertThat(opLatency).isEqualTo(2); + } + + @Test + public void testReadRowsAttemptLatency() throws InterruptedException { + final long sleepTime = 50; + final AtomicInteger callCount = new AtomicInteger(0); + + doAnswer( + new Answer() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + @SuppressWarnings("unchecked") + StreamObserver observer = + (StreamObserver) invocation.getArguments()[1]; + + Thread.sleep(sleepTime); + + // First attempt will return a transient error + if (callCount.getAndIncrement() == 0) { + observer.onError(new StatusRuntimeException(Status.UNAVAILABLE)); + return null; + } + // Next attempt will be ok + observer.onNext(DEFAULT_READ_ROWS_RESPONSES); + observer.onCompleted(); + return null; + } + }) + .when(mockService) + .readRows(any(ReadRowsRequest.class), anyObserver(ReadRowsResponse.class)); + + Stopwatch stopwatch = Stopwatch.createStarted(); + Lists.newArrayList(stub.readRowsCallable().call(Query.create(TABLE_ID))); + long elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS); + + // Give OpenCensus a chance to update the views asynchronously. + Thread.sleep(100); + + long attemptLatency = + getAggregationValueAsLong( + RpcViewConstants.BIGTABLE_ATTEMPT_LATENCY_VIEW, + ImmutableMap.of( + RpcMeasureConstants.BIGTABLE_OP, TagValue.create("Bigtable.ReadRows"), + RpcMeasureConstants.BIGTABLE_STATUS, TagValue.create("OK"))); + // Average attempt latency will be just a single wait (as opposed to op latency which will be 2x + // sleeptime) + assertThat(attemptLatency).isIn(Range.closed(sleepTime, elapsed - sleepTime)); + } + + @SuppressWarnings("unchecked") + private static StreamObserver anyObserver(Class returnType) { + return (StreamObserver) any(returnType); + } + + private long getAggregationValueAsLong(View view, ImmutableMap tags) { + ViewData viewData = localStats.getViewManager().getView(view.getName()); + Map, AggregationData> aggregationMap = + Objects.requireNonNull(viewData).getAggregationMap(); + + List tagValues = new ArrayList<>(); + + for (TagKey column : view.getColumns()) { + if (RpcMeasureConstants.BIGTABLE_PROJECT_ID == column) { + tagValues.add(TagValue.create(PROJECT_ID)); + } else if (RpcMeasureConstants.BIGTABLE_INSTANCE_ID == column) { + tagValues.add(TagValue.create(INSTANCE_ID)); + } else if (RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID == column) { + tagValues.add(TagValue.create(APP_PROFILE_ID)); + } else { + tagValues.add(tags.get(column)); + } + } + + AggregationData aggregationData = aggregationMap.get(tagValues); + + return aggregationData.match( + new Function() { + @Override + public Long apply(SumDataDouble arg) { + return (long) arg.getSum(); + } + }, + new Function() { + @Override + public Long apply(SumDataLong arg) { + return arg.getSum(); + } + }, + new Function() { + @Override + public Long apply(CountData arg) { + return arg.getCount(); + } + }, + new Function() { + @Override + public Long apply(DistributionData arg) { + return (long) arg.getMean(); + } + }, + new Function() { + @Override + public Long apply(LastValueDataDouble arg) { + return (long) arg.getLastValue(); + } + }, + new Function() { + @Override + public Long apply(LastValueDataLong arg) { + return arg.getLastValue(); + } + }, + new Function() { + @Override + public Long apply(AggregationData arg) { + throw new UnsupportedOperationException(); + } + }); + } +} diff --git a/grpc-google-cloud-bigtable-admin-v2/pom.xml b/grpc-google-cloud-bigtable-admin-v2/pom.xml index c4fc2ee5d7..05db558d03 100644 --- a/grpc-google-cloud-bigtable-admin-v2/pom.xml +++ b/grpc-google-cloud-bigtable-admin-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigtable-admin-v2 - 1.12.2 + 1.13.0 grpc-google-cloud-bigtable-admin-v2 GRPC library for grpc-google-cloud-bigtable-admin-v2 com.google.cloud google-cloud-bigtable-parent - 1.12.2 + 1.13.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.12.2 + 1.13.0 pom import com.google.cloud google-cloud-bigtable-bom - 1.12.2 + 1.13.0 pom import diff --git a/grpc-google-cloud-bigtable-v2/pom.xml b/grpc-google-cloud-bigtable-v2/pom.xml index b0bbacd3eb..c62e028096 100644 --- a/grpc-google-cloud-bigtable-v2/pom.xml +++ b/grpc-google-cloud-bigtable-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigtable-v2 - 1.12.2 + 1.13.0 grpc-google-cloud-bigtable-v2 GRPC library for grpc-google-cloud-bigtable-v2 com.google.cloud google-cloud-bigtable-parent - 1.12.2 + 1.13.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.12.2 + 1.13.0 pom import com.google.cloud google-cloud-bigtable-bom - 1.12.2 + 1.13.0 pom import diff --git a/pom.xml b/pom.xml index 97150db8fe..a1081ff3ca 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ google-cloud-bigtable-parent pom - 1.12.2 + 1.13.0 Google Cloud Bigtable Parent https://github.com/googleapis/java-bigtable @@ -14,7 +14,7 @@ com.google.cloud google-cloud-shared-config - 0.5.0 + 0.6.0 @@ -162,7 +162,7 @@ com.google.cloud google-cloud-conformance-tests - 0.0.10 + 0.0.11 + 1.13.0 proto-google-cloud-bigtable-admin-v2 PROTO library for proto-google-cloud-bigtable-admin-v2 com.google.cloud google-cloud-bigtable-parent - 1.12.2 + 1.13.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.12.2 + 1.13.0 pom import com.google.cloud google-cloud-bigtable-bom - 1.12.2 + 1.13.0 pom import diff --git a/proto-google-cloud-bigtable-v2/pom.xml b/proto-google-cloud-bigtable-v2/pom.xml index 8457275175..915d6c77fe 100644 --- a/proto-google-cloud-bigtable-v2/pom.xml +++ b/proto-google-cloud-bigtable-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigtable-v2 - 1.12.2 + 1.13.0 proto-google-cloud-bigtable-v2 PROTO library for proto-google-cloud-bigtable-v2 com.google.cloud google-cloud-bigtable-parent - 1.12.2 + 1.13.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.12.2 + 1.13.0 pom import com.google.cloud google-cloud-bigtable-bom - 1.12.2 + 1.13.0 pom import diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index 55b71b31fb..73f0482b65 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -29,7 +29,7 @@ com.google.cloud google-cloud-bigtable - 1.12.0 + 1.12.2 diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 8ee0d9486b..66cf071da4 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-bigtable - 1.12.2 + 1.13.0 diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index fadf0530e1..e36025e441 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -30,7 +30,7 @@ com.google.cloud libraries-bom - 5.3.0 + 5.4.0 pom import diff --git a/synth.metadata b/synth.metadata index 7d6d7b0273..926f340d0d 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,22 +4,22 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-bigtable.git", - "sha": "322fc6a39066ca13d3f523ff524c1c6c1d387bda" + "sha": "b17e39f2d8ca84335ab85d80372d6a0309a684c6" } }, { "git": { "name": "googleapis", "remote": "https://github.com/googleapis/googleapis.git", - "sha": "fb8f62b6784f43faf4b64179c57ce4b4931b1a00", - "internalRef": "310426317" + "sha": "a175708acd7a367ab3ed120fa562331e1761d825", + "internalRef": "312558019" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "be74d3e532faa47eb59f1a0eaebde0860d1d8ab4" + "sha": "98c50772ec23295c64cf0d2ddf199ea52961fd4c" } } ], diff --git a/versions.txt b/versions.txt index 05f38cf484..4bc33208e6 100644 --- a/versions.txt +++ b/versions.txt @@ -1,9 +1,9 @@ # Format: # module:released-version:current-version -google-cloud-bigtable:1.12.2:1.12.2 -grpc-google-cloud-bigtable-admin-v2:1.12.2:1.12.2 -grpc-google-cloud-bigtable-v2:1.12.2:1.12.2 -proto-google-cloud-bigtable-admin-v2:1.12.2:1.12.2 -proto-google-cloud-bigtable-v2:1.12.2:1.12.2 -google-cloud-bigtable-emulator:0.121.2:0.121.2 +google-cloud-bigtable:1.13.0:1.13.0 +grpc-google-cloud-bigtable-admin-v2:1.13.0:1.13.0 +grpc-google-cloud-bigtable-v2:1.13.0:1.13.0 +proto-google-cloud-bigtable-admin-v2:1.13.0:1.13.0 +proto-google-cloud-bigtable-v2:1.13.0:1.13.0 +google-cloud-bigtable-emulator:0.122.0:0.122.0