Exponential decaying average = weightedSum / weightedCount, where
+ * weightedSum(n) = weight(n) * value(n) + weightedSum(n - 1)
+ * weightedCount(n) = weight(n) + weightedCount(n - 1),
+ * and weight(n) grows exponentially over elapsed time. Biased to the past 5 minutes.
*/
final class DynamicFlowControlStats {
- private static final double DEFAULT_DECAY_CONSTANT = 0.015; // Biased to the past 5 minutes
+ // Biased to the past 5 minutes (300 seconds), e^(-decay_constant * 300) = 0.01, decay_constant ~=
+ // 0.015
+ private static final double DEFAULT_DECAY_CONSTANT = 0.015;
+ // Update decay cycle start time every 15 minutes so the values won't be infinite
+ private static final long DECAY_CYCLE_SECOND = TimeUnit.MINUTES.toSeconds(15);
- private AtomicLong lastAdjustedTimestampMs;
- private DecayingAverage meanLatency;
+ private final AtomicLong lastAdjustedTimestampMs;
+ private final DecayingAverage meanLatency;
DynamicFlowControlStats() {
- this(DEFAULT_DECAY_CONSTANT);
+ this(DEFAULT_DECAY_CONSTANT, NanoClock.getDefaultClock());
}
- DynamicFlowControlStats(double decayConstant) {
+ @InternalApi("visible for testing")
+ DynamicFlowControlStats(double decayConstant, ApiClock clock) {
this.lastAdjustedTimestampMs = new AtomicLong(0);
- this.meanLatency = new DecayingAverage(decayConstant);
+ this.meanLatency = new DecayingAverage(decayConstant, clock);
}
void updateLatency(long latency) {
- updateLatency(latency, System.currentTimeMillis());
- }
-
- @VisibleForTesting
- void updateLatency(long latency, long timestampMs) {
- meanLatency.update(latency, timestampMs);
+ meanLatency.update(latency);
}
+ /** Return the mean calculated from the last update, will not decay over time. */
double getMeanLatency() {
- return getMeanLatency(System.currentTimeMillis());
- }
-
- @VisibleForTesting
- double getMeanLatency(long timestampMs) {
- return meanLatency.getMean(timestampMs);
+ return meanLatency.getMean();
}
public long getLastAdjustedTimestampMs() {
@@ -71,46 +73,45 @@ private class DecayingAverage {
private double decayConstant;
private double mean;
private double weightedCount;
- private AtomicLong lastUpdateTimeInSecond;
+ private long decayCycleStartEpoch;
+ private final ApiClock clock;
- DecayingAverage(double decayConstant) {
+ DecayingAverage(double decayConstant, ApiClock clock) {
this.decayConstant = decayConstant;
this.mean = 0.0;
this.weightedCount = 0.0;
- this.lastUpdateTimeInSecond = new AtomicLong(0);
+ this.clock = clock;
+ this.decayCycleStartEpoch = TimeUnit.MILLISECONDS.toSeconds(clock.millisTime());
}
- synchronized void update(long value, long timestampMs) {
- long now = TimeUnit.MILLISECONDS.toSeconds(timestampMs);
- Preconditions.checkArgument(
- now >= lastUpdateTimeInSecond.get(), "can't update an event in the past");
- if (lastUpdateTimeInSecond.get() == 0) {
- lastUpdateTimeInSecond.set(now);
- mean = value;
- weightedCount = 1;
- } else {
- long prev = lastUpdateTimeInSecond.getAndSet(now);
- long elapsed = now - prev;
- double alpha = getAlpha(elapsed);
- // Exponential moving average = weightedSum / weightedCount, where
- // weightedSum(n) = value + alpha * weightedSum(n - 1)
- // weightedCount(n) = 1 + alpha * weightedCount(n - 1)
- // Using weighted count in case the sum overflows
- mean =
- mean * ((weightedCount * alpha) / (weightedCount * alpha + 1))
- + value / (weightedCount * alpha + 1);
- weightedCount = weightedCount * alpha + 1;
- }
+ synchronized void update(long value) {
+ long now = TimeUnit.MILLISECONDS.toSeconds(clock.millisTime());
+ double weight = getWeight(now);
+ // Using weighted count in case the sum overflows.
+ mean =
+ mean * (weightedCount / (weightedCount + weight))
+ + weight * value / (weightedCount + weight);
+ weightedCount += weight;
}
- double getMean(long timestampMs) {
- long timestampSecs = TimeUnit.MILLISECONDS.toSeconds(timestampMs);
- long elapsed = timestampSecs - lastUpdateTimeInSecond.get();
- return mean * getAlpha(Math.max(0, elapsed));
+ double getMean() {
+ return mean;
}
- private double getAlpha(long elapsedSecond) {
- return Math.exp(-decayConstant * elapsedSecond);
+ private double getWeight(long now) {
+ long elapsedSecond = now - decayCycleStartEpoch;
+ double weight = Math.exp(decayConstant * elapsedSecond);
+ // Decay mean, weightedCount and reset decay cycle start epoch every 15 minutes, so the
+ // values won't be infinite
+ if (elapsedSecond > DECAY_CYCLE_SECOND) {
+ mean /= weight;
+ weightedCount /= weight;
+ decayCycleStartEpoch = now;
+ // After resetting start time, weight = e^0 = 1
+ return 1;
+ } else {
+ return weight;
+ }
}
}
}
diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/DynamicFlowControlStatsTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/DynamicFlowControlStatsTest.java
index 653489f330..2a407dda93 100644
--- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/DynamicFlowControlStatsTest.java
+++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/DynamicFlowControlStatsTest.java
@@ -17,6 +17,7 @@
import static com.google.common.truth.Truth.assertThat;
+import com.google.api.core.ApiClock;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.ExecutionException;
@@ -24,43 +25,48 @@
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnit;
+import org.mockito.junit.MockitoRule;
@RunWith(JUnit4.class)
public class DynamicFlowControlStatsTest {
+ @Rule public final MockitoRule rule = MockitoJUnit.rule();
+
+ @Mock private ApiClock clock;
+
@Test
public void testUpdate() {
- DynamicFlowControlStats stats = new DynamicFlowControlStats();
- long now = System.currentTimeMillis();
- stats.updateLatency(10, now);
- assertThat(stats.getMeanLatency(now)).isEqualTo(10);
-
- stats.updateLatency(10, now);
- stats.updateLatency(10, now);
- assertThat(stats.getMeanLatency(now)).isEqualTo(10);
+ Mockito.when(clock.millisTime()).thenReturn(0L);
+ DynamicFlowControlStats stats = new DynamicFlowControlStats(0.015, clock);
+ stats.updateLatency(10);
+ assertThat(stats.getMeanLatency()).isEqualTo(10);
+ stats.updateLatency(10);
+ stats.updateLatency(10);
+ assertThat(stats.getMeanLatency()).isEqualTo(10);
// In five minutes the previous latency should be decayed to under 1. And the new average should
// be very close to 20
- long fiveMinutesLater = now + TimeUnit.MINUTES.toMillis(5);
- assertThat(stats.getMeanLatency(fiveMinutesLater)).isLessThan(1);
- stats.updateLatency(20, fiveMinutesLater);
- assertThat(stats.getMeanLatency(fiveMinutesLater)).isGreaterThan(19);
- assertThat(stats.getMeanLatency(fiveMinutesLater)).isLessThan(20);
-
- long aDayLater = now + TimeUnit.HOURS.toMillis(24);
- assertThat(stats.getMeanLatency(aDayLater)).isZero();
+ Mockito.when(clock.millisTime()).thenReturn(TimeUnit.MINUTES.toMillis(5));
+ stats.updateLatency(20);
+ assertThat(stats.getMeanLatency()).isGreaterThan(19);
+ assertThat(stats.getMeanLatency()).isLessThan(20);
- long timestamp = aDayLater;
+ // After a day
+ long aDay = TimeUnit.DAYS.toMillis(1);
for (int i = 0; i < 10; i++) {
- timestamp += TimeUnit.SECONDS.toMillis(i);
- stats.updateLatency(i, timestamp);
+ Mockito.when(clock.millisTime()).thenReturn(aDay + TimeUnit.SECONDS.toMillis(i));
+ stats.updateLatency(i);
}
- assertThat(stats.getMeanLatency(timestamp)).isGreaterThan(4.5);
- assertThat(stats.getMeanLatency(timestamp)).isLessThan(6);
+ assertThat(stats.getMeanLatency()).isGreaterThan(4.5);
+ assertThat(stats.getMeanLatency()).isLessThan(6);
}
@Test(timeout = 1000)
From 9b3c6013fef396fcc923e53c13f73a03a48c9c02 Mon Sep 17 00:00:00 2001
From: Igor Bernstein
Date: Thu, 3 Jun 2021 12:00:24 -0400
Subject: [PATCH 18/19] feat: promote stream wait timeouts to deadlines for
point reads (#848)
Special case point reads to use grpc's deadlines instead of relying on the watchdog
---
.../data/v2/stub/EnhancedBigtableStub.java | 9 +-
.../readrows/PointReadTimeoutCallable.java | 86 ++++++++
.../PointReadTimeoutCallableTest.java | 183 ++++++++++++++++++
3 files changed, 276 insertions(+), 2 deletions(-)
create mode 100644 google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/readrows/PointReadTimeoutCallable.java
create mode 100644 google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/readrows/PointReadTimeoutCallableTest.java
diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java
index c08f0aec1e..55e928d59f 100644
--- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java
+++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java
@@ -75,6 +75,7 @@
import com.google.cloud.bigtable.data.v2.stub.mutaterows.MutateRowsBatchingDescriptor;
import com.google.cloud.bigtable.data.v2.stub.mutaterows.MutateRowsRetryingCallable;
import com.google.cloud.bigtable.data.v2.stub.readrows.FilterMarkerRowsCallable;
+import com.google.cloud.bigtable.data.v2.stub.readrows.PointReadTimeoutCallable;
import com.google.cloud.bigtable.data.v2.stub.readrows.ReadRowsBatchingDescriptor;
import com.google.cloud.bigtable.data.v2.stub.readrows.ReadRowsConvertExceptionCallable;
import com.google.cloud.bigtable.data.v2.stub.readrows.ReadRowsResumptionStrategy;
@@ -336,7 +337,7 @@ public UnaryCallable createReadRowCallable(RowAdapter
private ServerStreamingCallable createReadRowsBaseCallable(
ServerStreamingCallSettings readRowsSettings, RowAdapter rowAdapter) {
- ServerStreamingCallable base =
+ final ServerStreamingCallable base =
GrpcRawCallableFactory.createServerStreamingCallable(
GrpcCallSettings.newBuilder()
.setMethodDescriptor(BigtableGrpc.getReadRowsMethod())
@@ -352,11 +353,15 @@ public Map extract(ReadRowsRequest readRowsRequest) {
.build(),
readRowsSettings.getRetryableCodes());
+ // Promote streamWaitTimeout to deadline for point reads
+ ServerStreamingCallable withPointTimeouts =
+ new PointReadTimeoutCallable<>(base);
+
// Sometimes ReadRows connections are disconnected via an RST frame. This error is transient and
// should be treated similar to UNAVAILABLE. However, this exception has an INTERNAL error code
// which by default is not retryable. Convert the exception so it can be retried in the client.
ServerStreamingCallable convertException =
- new ReadRowsConvertExceptionCallable<>(base);
+ new ReadRowsConvertExceptionCallable<>(withPointTimeouts);
ServerStreamingCallable merging =
new RowMergingCallable<>(convertException, rowAdapter);
diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/readrows/PointReadTimeoutCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/readrows/PointReadTimeoutCallable.java
new file mode 100644
index 0000000000..7ce0e8b7c6
--- /dev/null
+++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/readrows/PointReadTimeoutCallable.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.bigtable.data.v2.stub.readrows;
+
+import com.google.api.core.InternalApi;
+import com.google.api.gax.rpc.ApiCallContext;
+import com.google.api.gax.rpc.ResponseObserver;
+import com.google.api.gax.rpc.ServerStreamingCallable;
+import com.google.bigtable.v2.ReadRowsRequest;
+import javax.annotation.Nullable;
+import org.threeten.bp.Duration;
+
+/**
+ * Specialization of ReadRows streams for point reads.
+ *
+ * Under normal circumstances, the ReadRows RPC can't make any assumptions about deadlines. In
+ * general case the end user can be issuing a full table scan. However, when dealing with point
+ * reads, the client can make assumptions and promote the per row timeout to be a per attempt
+ * timeout.
+ *
+ *
This callable will check if the request is a point read and promote the timeout to be a
+ * deadline.
+ */
+@InternalApi
+public class PointReadTimeoutCallable
+ extends ServerStreamingCallable {
+ private final ServerStreamingCallable inner;
+
+ public PointReadTimeoutCallable(ServerStreamingCallable inner) {
+ this.inner = inner;
+ }
+
+ @Override
+ public void call(ReadRowsRequest request, ResponseObserver observer, ApiCallContext ctx) {
+ if (isPointRead(request)) {
+ Duration effectiveTimeout = getEffectivePointReadTimeout(ctx);
+ if (effectiveTimeout != null) {
+ ctx = ctx.withTimeout(effectiveTimeout);
+ }
+ }
+ inner.call(request, observer, ctx);
+ }
+
+ private boolean isPointRead(ReadRowsRequest request) {
+ if (request.getRowsLimit() == 1) {
+ return true;
+ }
+ if (!request.getRows().getRowRangesList().isEmpty()) {
+ return false;
+ }
+ return request.getRows().getRowKeysCount() == 1;
+ }
+
+ /**
+ * Extracts the effective timeout for a point read.
+ *
+ * The effective time is the minimum of a streamWaitTimeout and a user set attempt timeout.
+ */
+ @Nullable
+ private Duration getEffectivePointReadTimeout(ApiCallContext ctx) {
+ Duration streamWaitTimeout = ctx.getStreamWaitTimeout();
+ Duration attemptTimeout = ctx.getTimeout();
+
+ if (streamWaitTimeout == null) {
+ return attemptTimeout;
+ }
+
+ if (attemptTimeout == null) {
+ return streamWaitTimeout;
+ }
+ return (attemptTimeout.compareTo(streamWaitTimeout) <= 0) ? attemptTimeout : streamWaitTimeout;
+ }
+}
diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/readrows/PointReadTimeoutCallableTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/readrows/PointReadTimeoutCallableTest.java
new file mode 100644
index 0000000000..a3941cd5c1
--- /dev/null
+++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/readrows/PointReadTimeoutCallableTest.java
@@ -0,0 +1,183 @@
+/*
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.bigtable.data.v2.stub.readrows;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.api.gax.grpc.GrpcCallContext;
+import com.google.api.gax.rpc.ApiCallContext;
+import com.google.api.gax.rpc.ResponseObserver;
+import com.google.api.gax.rpc.ServerStreamingCallable;
+import com.google.bigtable.v2.ReadRowsRequest;
+import com.google.bigtable.v2.RowRange;
+import com.google.bigtable.v2.RowSet;
+import com.google.common.collect.ImmutableList;
+import com.google.protobuf.ByteString;
+import java.util.Arrays;
+import java.util.List;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnit;
+import org.mockito.junit.MockitoRule;
+import org.threeten.bp.Duration;
+
+@RunWith(JUnit4.class)
+public class PointReadTimeoutCallableTest {
+ @Rule public final MockitoRule moo = MockitoJUnit.rule();
+
+ @Mock private ServerStreamingCallable inner;
+ private ArgumentCaptor ctxCaptor;
+ @Mock private ResponseObserver