From eca8bf8e62ae76cb93e2ba7059628c08da49b935 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Wed, 26 Aug 2020 15:27:28 -0700 Subject: [PATCH 01/86] Start 1.33.0 development cycle --- build.gradle | 2 +- .../src/test/golden/TestDeprecatedService.java.txt | 2 +- compiler/src/test/golden/TestService.java.txt | 2 +- .../src/testLite/golden/TestDeprecatedService.java.txt | 2 +- compiler/src/testLite/golden/TestService.java.txt | 2 +- core/src/main/java/io/grpc/internal/GrpcUtil.java | 2 +- examples/android/clientcache/app/build.gradle | 10 +++++----- examples/android/helloworld/app/build.gradle | 8 ++++---- examples/android/routeguide/app/build.gradle | 8 ++++---- examples/android/strictmode/app/build.gradle | 8 ++++---- examples/build.gradle | 2 +- examples/example-alts/build.gradle | 2 +- examples/example-gauth/build.gradle | 2 +- examples/example-gauth/pom.xml | 4 ++-- examples/example-hostname/build.gradle | 2 +- examples/example-hostname/pom.xml | 4 ++-- examples/example-jwt-auth/build.gradle | 2 +- examples/example-jwt-auth/pom.xml | 4 ++-- examples/example-tls/build.gradle | 2 +- examples/example-tls/pom.xml | 4 ++-- examples/example-xds/build.gradle | 2 +- examples/pom.xml | 4 ++-- 22 files changed, 40 insertions(+), 40 deletions(-) diff --git a/build.gradle b/build.gradle index fcb2ddd6678..df5826c8628 100644 --- a/build.gradle +++ b/build.gradle @@ -18,7 +18,7 @@ subprojects { apply plugin: "net.ltgt.errorprone" group = "io.grpc" - version = "1.32.0-SNAPSHOT" // CURRENT_GRPC_VERSION + version = "1.33.0-SNAPSHOT" // CURRENT_GRPC_VERSION repositories { maven { // The google mirror is less flaky than mavenCentral() diff --git a/compiler/src/test/golden/TestDeprecatedService.java.txt b/compiler/src/test/golden/TestDeprecatedService.java.txt index 1374659889b..edceeae5cb6 100644 --- a/compiler/src/test/golden/TestDeprecatedService.java.txt +++ b/compiler/src/test/golden/TestDeprecatedService.java.txt @@ -21,7 +21,7 @@ import static io.grpc.stub.ServerCalls.asyncUnimplementedUnaryCall; * */ @javax.annotation.Generated( - value = "by gRPC proto compiler (version 1.32.0-SNAPSHOT)", + value = "by gRPC proto compiler (version 1.33.0-SNAPSHOT)", comments = "Source: grpc/testing/compiler/test.proto") @java.lang.Deprecated public final class TestDeprecatedServiceGrpc { diff --git a/compiler/src/test/golden/TestService.java.txt b/compiler/src/test/golden/TestService.java.txt index f6fd217cbdd..033c93175dd 100644 --- a/compiler/src/test/golden/TestService.java.txt +++ b/compiler/src/test/golden/TestService.java.txt @@ -21,7 +21,7 @@ import static io.grpc.stub.ServerCalls.asyncUnimplementedUnaryCall; * */ @javax.annotation.Generated( - value = "by gRPC proto compiler (version 1.32.0-SNAPSHOT)", + value = "by gRPC proto compiler (version 1.33.0-SNAPSHOT)", comments = "Source: grpc/testing/compiler/test.proto") public final class TestServiceGrpc { diff --git a/compiler/src/testLite/golden/TestDeprecatedService.java.txt b/compiler/src/testLite/golden/TestDeprecatedService.java.txt index 694bab3b3d8..b7ce089ca2f 100644 --- a/compiler/src/testLite/golden/TestDeprecatedService.java.txt +++ b/compiler/src/testLite/golden/TestDeprecatedService.java.txt @@ -21,7 +21,7 @@ import static io.grpc.stub.ServerCalls.asyncUnimplementedUnaryCall; * */ @javax.annotation.Generated( - value = "by gRPC proto compiler (version 1.32.0-SNAPSHOT)", + value = "by gRPC proto compiler (version 1.33.0-SNAPSHOT)", comments = "Source: grpc/testing/compiler/test.proto") @java.lang.Deprecated public final class TestDeprecatedServiceGrpc { diff --git a/compiler/src/testLite/golden/TestService.java.txt b/compiler/src/testLite/golden/TestService.java.txt index 50fc255cd4a..4693d662b2c 100644 --- a/compiler/src/testLite/golden/TestService.java.txt +++ b/compiler/src/testLite/golden/TestService.java.txt @@ -21,7 +21,7 @@ import static io.grpc.stub.ServerCalls.asyncUnimplementedUnaryCall; * */ @javax.annotation.Generated( - value = "by gRPC proto compiler (version 1.32.0-SNAPSHOT)", + value = "by gRPC proto compiler (version 1.33.0-SNAPSHOT)", comments = "Source: grpc/testing/compiler/test.proto") public final class TestServiceGrpc { diff --git a/core/src/main/java/io/grpc/internal/GrpcUtil.java b/core/src/main/java/io/grpc/internal/GrpcUtil.java index 3e210961541..18a3ebb7271 100644 --- a/core/src/main/java/io/grpc/internal/GrpcUtil.java +++ b/core/src/main/java/io/grpc/internal/GrpcUtil.java @@ -197,7 +197,7 @@ public byte[] parseAsciiString(byte[] serialized) { public static final Splitter ACCEPT_ENCODING_SPLITTER = Splitter.on(',').trimResults(); - private static final String IMPLEMENTATION_VERSION = "1.32.0-SNAPSHOT"; // CURRENT_GRPC_VERSION + private static final String IMPLEMENTATION_VERSION = "1.33.0-SNAPSHOT"; // CURRENT_GRPC_VERSION /** * The default timeout in nanos for a keepalive ping request. diff --git a/examples/android/clientcache/app/build.gradle b/examples/android/clientcache/app/build.gradle index 96e05eba65f..084cc22aba0 100644 --- a/examples/android/clientcache/app/build.gradle +++ b/examples/android/clientcache/app/build.gradle @@ -30,7 +30,7 @@ android { protobuf { protoc { artifact = 'com.google.protobuf:protoc:3.12.0' } plugins { - grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION + grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION } } generateProtoTasks { @@ -50,12 +50,12 @@ dependencies { implementation 'com.android.support:appcompat-v7:27.0.2' // You need to build grpc-java to obtain these libraries below. - implementation 'io.grpc:grpc-okhttp:1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-protobuf-lite:1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-stub:1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-okhttp:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-protobuf-lite:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-stub:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION implementation 'org.apache.tomcat:annotations-api:6.0.53' testImplementation 'junit:junit:4.12' testImplementation 'com.google.truth:truth:1.0.1' - testImplementation 'io.grpc:grpc-testing:1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION + testImplementation 'io.grpc:grpc-testing:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION } diff --git a/examples/android/helloworld/app/build.gradle b/examples/android/helloworld/app/build.gradle index 6f0b4da88a9..ab300b8428f 100644 --- a/examples/android/helloworld/app/build.gradle +++ b/examples/android/helloworld/app/build.gradle @@ -28,7 +28,7 @@ android { protobuf { protoc { artifact = 'com.google.protobuf:protoc:3.12.0' } plugins { - grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION + grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION } } generateProtoTasks { @@ -48,8 +48,8 @@ dependencies { implementation 'com.android.support:appcompat-v7:27.0.2' // You need to build grpc-java to obtain these libraries below. - implementation 'io.grpc:grpc-okhttp:1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-protobuf-lite:1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-stub:1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-okhttp:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-protobuf-lite:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-stub:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION implementation 'org.apache.tomcat:annotations-api:6.0.53' } diff --git a/examples/android/routeguide/app/build.gradle b/examples/android/routeguide/app/build.gradle index 3c32ad0f46f..e1a9d728800 100644 --- a/examples/android/routeguide/app/build.gradle +++ b/examples/android/routeguide/app/build.gradle @@ -28,7 +28,7 @@ android { protobuf { protoc { artifact = 'com.google.protobuf:protoc:3.12.0' } plugins { - grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION + grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION } } generateProtoTasks { @@ -48,8 +48,8 @@ dependencies { implementation 'com.android.support:appcompat-v7:27.0.2' // You need to build grpc-java to obtain these libraries below. - implementation 'io.grpc:grpc-okhttp:1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-protobuf-lite:1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-stub:1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-okhttp:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-protobuf-lite:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-stub:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION implementation 'org.apache.tomcat:annotations-api:6.0.53' } diff --git a/examples/android/strictmode/app/build.gradle b/examples/android/strictmode/app/build.gradle index 03d923d5a80..35cca7019a9 100644 --- a/examples/android/strictmode/app/build.gradle +++ b/examples/android/strictmode/app/build.gradle @@ -29,7 +29,7 @@ android { protobuf { protoc { artifact = 'com.google.protobuf:protoc:3.12.0' } plugins { - grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION + grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION } } generateProtoTasks { @@ -49,8 +49,8 @@ dependencies { implementation 'com.android.support:appcompat-v7:28.0.0' // You need to build grpc-java to obtain these libraries below. - implementation 'io.grpc:grpc-okhttp:1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-protobuf-lite:1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-stub:1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-okhttp:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-protobuf-lite:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-stub:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION implementation 'org.apache.tomcat:annotations-api:6.0.53' } diff --git a/examples/build.gradle b/examples/build.gradle index 770e9e28d9e..68bba439d65 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -22,7 +22,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION def protobufVersion = '3.12.0' def protocVersion = protobufVersion diff --git a/examples/example-alts/build.gradle b/examples/example-alts/build.gradle index 7f3d202f01f..7b34c2da0e2 100644 --- a/examples/example-alts/build.gradle +++ b/examples/example-alts/build.gradle @@ -23,7 +23,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION def protocVersion = '3.12.0' dependencies { diff --git a/examples/example-gauth/build.gradle b/examples/example-gauth/build.gradle index 76a1563b167..0a2a92d8fac 100644 --- a/examples/example-gauth/build.gradle +++ b/examples/example-gauth/build.gradle @@ -23,7 +23,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION def protobufVersion = '3.12.0' def protocVersion = protobufVersion diff --git a/examples/example-gauth/pom.xml b/examples/example-gauth/pom.xml index 779f91a0250..9c81ea6d1af 100644 --- a/examples/example-gauth/pom.xml +++ b/examples/example-gauth/pom.xml @@ -6,13 +6,13 @@ jar - 1.32.0-SNAPSHOT + 1.33.0-SNAPSHOT example-gauth https://github.com/grpc/grpc-java UTF-8 - 1.32.0-SNAPSHOT + 1.33.0-SNAPSHOT 3.12.0 1.7 diff --git a/examples/example-hostname/build.gradle b/examples/example-hostname/build.gradle index b7624483aac..2515e189d64 100644 --- a/examples/example-hostname/build.gradle +++ b/examples/example-hostname/build.gradle @@ -21,7 +21,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION def protobufVersion = '3.12.0' dependencies { diff --git a/examples/example-hostname/pom.xml b/examples/example-hostname/pom.xml index fea6ee83fd0..9b616a0a3fb 100644 --- a/examples/example-hostname/pom.xml +++ b/examples/example-hostname/pom.xml @@ -6,13 +6,13 @@ jar - 1.32.0-SNAPSHOT + 1.33.0-SNAPSHOT example-hostname https://github.com/grpc/grpc-java UTF-8 - 1.32.0-SNAPSHOT + 1.33.0-SNAPSHOT 3.12.0 1.7 diff --git a/examples/example-jwt-auth/build.gradle b/examples/example-jwt-auth/build.gradle index a597a7e4beb..588daf3794a 100644 --- a/examples/example-jwt-auth/build.gradle +++ b/examples/example-jwt-auth/build.gradle @@ -22,7 +22,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION def protobufVersion = '3.12.0' def protocVersion = protobufVersion diff --git a/examples/example-jwt-auth/pom.xml b/examples/example-jwt-auth/pom.xml index cc813513bec..8aa08257741 100644 --- a/examples/example-jwt-auth/pom.xml +++ b/examples/example-jwt-auth/pom.xml @@ -7,13 +7,13 @@ jar - 1.32.0-SNAPSHOT + 1.33.0-SNAPSHOT example-jwt-auth https://github.com/grpc/grpc-java UTF-8 - 1.32.0-SNAPSHOT + 1.33.0-SNAPSHOT 3.12.0 3.12.0 diff --git a/examples/example-tls/build.gradle b/examples/example-tls/build.gradle index 9b420626e71..fbec7406092 100644 --- a/examples/example-tls/build.gradle +++ b/examples/example-tls/build.gradle @@ -23,7 +23,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION def nettyTcNativeVersion = '2.0.31.Final' def protocVersion = '3.12.0' diff --git a/examples/example-tls/pom.xml b/examples/example-tls/pom.xml index e51be823719..fbbad150547 100644 --- a/examples/example-tls/pom.xml +++ b/examples/example-tls/pom.xml @@ -6,13 +6,13 @@ jar - 1.32.0-SNAPSHOT + 1.33.0-SNAPSHOT example-tls https://github.com/grpc/grpc-java UTF-8 - 1.32.0-SNAPSHOT + 1.33.0-SNAPSHOT 3.12.0 2.0.31.Final diff --git a/examples/example-xds/build.gradle b/examples/example-xds/build.gradle index 4142790c764..fab5c9452b2 100644 --- a/examples/example-xds/build.gradle +++ b/examples/example-xds/build.gradle @@ -18,7 +18,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.32.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION dependencies { // This example's client is the same as the helloworld client. We depend on the helloworld diff --git a/examples/pom.xml b/examples/pom.xml index d6aa9b6aabc..843d7da2adb 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -6,13 +6,13 @@ jar - 1.32.0-SNAPSHOT + 1.33.0-SNAPSHOT examples https://github.com/grpc/grpc-java UTF-8 - 1.32.0-SNAPSHOT + 1.33.0-SNAPSHOT 3.12.0 3.12.0 From 24a1095fd8f9f4e438e6d1554e86aeff53415849 Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Wed, 26 Aug 2020 22:22:25 -0700 Subject: [PATCH 02/86] core: refactor to delete RetryPolicy.DEFAULT We used `null` and `RetryPolicy.DEFAULT` for the value of `retryPolicy` in `RetriableStream` to distinguish the state between name resolution not being completed and name resolution being completed but no retry policy. After the change #7259 , name resolution is always completed when creating a `RetriableStream`, so the distinction will be gone. It will be cleaner to get rid of `RetryPolicy.DEFAULT` and simply use `null` for absence of RetryPolicy. `RetryPolicy.Provider` will be deleted in upcoming PR. --- .../java/io/grpc/internal/HedgingPolicy.java | 11 +++++----- .../internal/ManagedChannelServiceConfig.java | 4 ++-- .../io/grpc/internal/RetriableStream.java | 22 ++++++++++--------- .../java/io/grpc/internal/RetryPolicy.java | 12 +++++----- .../internal/ServiceConfigInterceptor.java | 17 ++++++++------ .../io/grpc/internal/HedgingPolicyTest.java | 12 +++------- .../io/grpc/internal/RetriableStreamTest.java | 10 ++++----- .../io/grpc/internal/RetryPolicyTest.java | 12 +++------- .../ServiceConfigInterceptorTest.java | 4 ++-- 9 files changed, 47 insertions(+), 57 deletions(-) diff --git a/core/src/main/java/io/grpc/internal/HedgingPolicy.java b/core/src/main/java/io/grpc/internal/HedgingPolicy.java index 6f7035020c1..83d1574961c 100644 --- a/core/src/main/java/io/grpc/internal/HedgingPolicy.java +++ b/core/src/main/java/io/grpc/internal/HedgingPolicy.java @@ -20,8 +20,8 @@ import com.google.common.base.Objects; import com.google.common.collect.ImmutableSet; import io.grpc.Status.Code; -import java.util.Collections; import java.util.Set; +import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; /** @@ -33,10 +33,6 @@ final class HedgingPolicy { final long hedgingDelayNanos; final Set nonFatalStatusCodes; - /** No hedging. */ - static final HedgingPolicy DEFAULT = - new HedgingPolicy(1, 0, Collections.emptySet()); - /** * The caller is supposed to have validated the arguments and handled throwing exception or * logging warnings already, so we avoid repeating args check here. @@ -75,14 +71,17 @@ public String toString() { .toString(); } + // TODO(zdapeng): delete this because HedgingPolicy will be always available prior to starting + // RetriableStream. /** * Provides the most suitable hedging policy for a call. */ interface Provider { /** - * This method is used no more than once for each call. Never returns null. + * This method is used no more than once for each call. */ + @Nullable HedgingPolicy get(); } } diff --git a/core/src/main/java/io/grpc/internal/ManagedChannelServiceConfig.java b/core/src/main/java/io/grpc/internal/ManagedChannelServiceConfig.java index fc78d2231a0..4c64311ce89 100644 --- a/core/src/main/java/io/grpc/internal/ManagedChannelServiceConfig.java +++ b/core/src/main/java/io/grpc/internal/ManagedChannelServiceConfig.java @@ -267,12 +267,12 @@ static final class MethodInfo { Map retryPolicyMap = retryEnabled ? ServiceConfigUtil.getRetryPolicyFromMethodConfig(methodConfig) : null; retryPolicy = retryPolicyMap == null - ? RetryPolicy.DEFAULT : retryPolicy(retryPolicyMap, maxRetryAttemptsLimit); + ? null : retryPolicy(retryPolicyMap, maxRetryAttemptsLimit); Map hedgingPolicyMap = retryEnabled ? ServiceConfigUtil.getHedgingPolicyFromMethodConfig(methodConfig) : null; hedgingPolicy = hedgingPolicyMap == null - ? HedgingPolicy.DEFAULT : hedgingPolicy(hedgingPolicyMap, maxHedgedAttemptsLimit); + ? null : hedgingPolicy(hedgingPolicyMap, maxHedgedAttemptsLimit); } @Override diff --git a/core/src/main/java/io/grpc/internal/RetriableStream.java b/core/src/main/java/io/grpc/internal/RetriableStream.java index 44f194c5d0a..c4dccf04c53 100644 --- a/core/src/main/java/io/grpc/internal/RetriableStream.java +++ b/core/src/main/java/io/grpc/internal/RetriableStream.java @@ -67,7 +67,9 @@ abstract class RetriableStream implements ClientStream { private final Metadata headers; private final RetryPolicy.Provider retryPolicyProvider; private final HedgingPolicy.Provider hedgingPolicyProvider; + @Nullable private RetryPolicy retryPolicy; + @Nullable private HedgingPolicy hedgingPolicy; private boolean isHedging; @@ -315,12 +317,9 @@ public void runWith(Substream substream) { Substream substream = createSubstream(0); checkState(hedgingPolicy == null, "hedgingPolicy has been initialized unexpectedly"); - // TODO(zdapeng): if substream is a DelayedStream, do this when name resolution finishes hedgingPolicy = hedgingPolicyProvider.get(); - if (!HedgingPolicy.DEFAULT.equals(hedgingPolicy)) { + if (null != hedgingPolicy) { isHedging = true; - retryPolicy = RetryPolicy.DEFAULT; - FutureCanceller scheduledHedgingRef = null; synchronized (lock) { @@ -810,7 +809,7 @@ public void closed(Status status, RpcProgress rpcProgress, Metadata trailers) { if (retryPolicy == null) { retryPolicy = retryPolicyProvider.get(); } - if (retryPolicy.maxAttempts == 1) { + if (retryPolicy == null || retryPolicy.maxAttempts == 1) { // optimization for early commit commitAndRun(newSubstream); } @@ -831,11 +830,6 @@ public void run() { } else { noMoreTransparentRetry.set(true); - if (retryPolicy == null) { - retryPolicy = retryPolicyProvider.get(); - nextBackoffIntervalNanos = retryPolicy.initialBackoffNanos; - } - if (isHedging) { HedgingPlan hedgingPlan = makeHedgingDecision(status, trailers); if (hedgingPlan.isHedgeable) { @@ -900,6 +894,14 @@ public void run() { * caller should check it separately. It also updates the throttle. It does not change state. */ private RetryPlan makeRetryDecision(Status status, Metadata trailer) { + if (retryPolicy == null) { + retryPolicy = retryPolicyProvider.get(); + if (retryPolicy == null) { + return new RetryPlan(false, 0); + } else { + nextBackoffIntervalNanos = retryPolicy.initialBackoffNanos; + } + } boolean shouldRetry = false; long backoffNanos = 0L; boolean isRetryableStatusCode = retryPolicy.retryableStatusCodes.contains(status.getCode()); diff --git a/core/src/main/java/io/grpc/internal/RetryPolicy.java b/core/src/main/java/io/grpc/internal/RetryPolicy.java index 928e5dc7fc0..c26e06d3376 100644 --- a/core/src/main/java/io/grpc/internal/RetryPolicy.java +++ b/core/src/main/java/io/grpc/internal/RetryPolicy.java @@ -19,11 +19,10 @@ import com.google.common.base.MoreObjects; import com.google.common.base.Objects; import com.google.common.collect.ImmutableSet; -import io.grpc.Status; import io.grpc.Status.Code; -import java.util.Collections; import java.util.Set; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; /** @@ -37,10 +36,6 @@ final class RetryPolicy { final double backoffMultiplier; final Set retryableStatusCodes; - /** No retry. */ - static final RetryPolicy DEFAULT = - new RetryPolicy(1, 0, 0, 1, Collections.emptySet()); - /** * The caller is supposed to have validated the arguments and handled throwing exception or * logging warnings already, so we avoid repeating args check here. @@ -92,14 +87,17 @@ public String toString() { .toString(); } + // TODO(zdapeng): delete this because RetryPolicy will be always available prior to starting + // RetriableStream. /** * Provides the most suitable retry policy for a call. */ interface Provider { /** - * This method is used no more than once for each call. Never returns null. + * This method is used no more than once for each call. */ + @Nullable RetryPolicy get(); } } diff --git a/core/src/main/java/io/grpc/internal/ServiceConfigInterceptor.java b/core/src/main/java/io/grpc/internal/ServiceConfigInterceptor.java index 9fb91db5ea7..4ffc25d5d4f 100644 --- a/core/src/main/java/io/grpc/internal/ServiceConfigInterceptor.java +++ b/core/src/main/java/io/grpc/internal/ServiceConfigInterceptor.java @@ -82,7 +82,7 @@ public HedgingPolicy get() { } verify( - retryPolicy.equals(RetryPolicy.DEFAULT) || hedgingPolicy.equals(HedgingPolicy.DEFAULT), + retryPolicy == null || hedgingPolicy == null, "Can not apply both retry and hedging policy for the method '%s'", method); callOptions = callOptions @@ -97,9 +97,10 @@ final class DelayedRetryPolicyProvider implements RetryPolicy.Provider { *

Note that this method is used no more than once for each call. */ @Override + @Nullable public RetryPolicy get() { if (!initComplete) { - return RetryPolicy.DEFAULT; + return null; } return getRetryPolicyFromConfig(method); } @@ -113,14 +114,14 @@ final class DelayedHedgingPolicyProvider implements HedgingPolicy.Provider { *

Note that this method is used no more than once for each call. */ @Override + @Nullable public HedgingPolicy get() { if (!initComplete) { - return HedgingPolicy.DEFAULT; + return null; } HedgingPolicy hedgingPolicy = getHedgingPolicyFromConfig(method); verify( - hedgingPolicy.equals(HedgingPolicy.DEFAULT) - || getRetryPolicyFromConfig(method).equals(RetryPolicy.DEFAULT), + hedgingPolicy == null || getRetryPolicyFromConfig(method) == null, "Can not apply both retry and hedging policy for the method '%s'", method); return hedgingPolicy; } @@ -190,14 +191,16 @@ private MethodInfo getMethodInfo(MethodDescriptor method) { } @VisibleForTesting + @Nullable RetryPolicy getRetryPolicyFromConfig(MethodDescriptor method) { MethodInfo info = getMethodInfo(method); - return info == null ? RetryPolicy.DEFAULT : info.retryPolicy; + return info == null ? null : info.retryPolicy; } @VisibleForTesting + @Nullable HedgingPolicy getHedgingPolicyFromConfig(MethodDescriptor method) { MethodInfo info = getMethodInfo(method); - return info == null ? HedgingPolicy.DEFAULT : info.hedgingPolicy; + return info == null ? null : info.hedgingPolicy; } } diff --git a/core/src/test/java/io/grpc/internal/HedgingPolicyTest.java b/core/src/test/java/io/grpc/internal/HedgingPolicyTest.java index 7c640862f8a..1a7cab8e70e 100644 --- a/core/src/test/java/io/grpc/internal/HedgingPolicyTest.java +++ b/core/src/test/java/io/grpc/internal/HedgingPolicyTest.java @@ -74,14 +74,10 @@ public void getHedgingPolicies() throws Exception { MethodDescriptor.Builder builder = TestMethodDescriptors.voidMethod().toBuilder(); MethodDescriptor method = builder.setFullMethodName("not/exist").build(); - assertEquals( - HedgingPolicy.DEFAULT, - serviceConfigInterceptor.getHedgingPolicyFromConfig(method)); + assertThat(serviceConfigInterceptor.getHedgingPolicyFromConfig(method)).isNull(); method = builder.setFullMethodName("not_exist/Foo1").build(); - assertEquals( - HedgingPolicy.DEFAULT, - serviceConfigInterceptor.getHedgingPolicyFromConfig(method)); + assertThat(serviceConfigInterceptor.getHedgingPolicyFromConfig(method)).isNull(); method = builder.setFullMethodName("SimpleService1/not_exist").build(); @@ -101,9 +97,7 @@ public void getHedgingPolicies() throws Exception { serviceConfigInterceptor.getHedgingPolicyFromConfig(method)); method = builder.setFullMethodName("SimpleService2/not_exist").build(); - assertEquals( - HedgingPolicy.DEFAULT, - serviceConfigInterceptor.getHedgingPolicyFromConfig(method)); + assertThat(serviceConfigInterceptor.getHedgingPolicyFromConfig(method)).isNull(); method = builder.setFullMethodName("SimpleService2/Foo2").build(); assertEquals( diff --git a/core/src/test/java/io/grpc/internal/RetriableStreamTest.java b/core/src/test/java/io/grpc/internal/RetriableStreamTest.java index 8a69fe501b2..5aab2f0856a 100644 --- a/core/src/test/java/io/grpc/internal/RetriableStreamTest.java +++ b/core/src/test/java/io/grpc/internal/RetriableStreamTest.java @@ -144,8 +144,8 @@ private final class RecordedRetriableStream extends RetriableStream { ChannelBufferMeter channelBufferUsed, long perRpcBufferLimit, long channelBufferLimit, Executor callExecutor, ScheduledExecutorService scheduledExecutorService, - final RetryPolicy retryPolicy, - final HedgingPolicy hedgingPolicy, + @Nullable final RetryPolicy retryPolicy, + @Nullable final HedgingPolicy hedgingPolicy, @Nullable Throttle throttle) { super( method, headers, channelBufferUsed, perRpcBufferLimit, channelBufferLimit, callExecutor, @@ -196,14 +196,14 @@ private RetriableStream newThrottledRetriableStream(Throttle throttle) { return new RecordedRetriableStream( method, new Metadata(), channelBufferUsed, PER_RPC_BUFFER_LIMIT, CHANNEL_BUFFER_LIMIT, MoreExecutors.directExecutor(), fakeClock.getScheduledExecutorService(), RETRY_POLICY, - HedgingPolicy.DEFAULT, throttle); + null, throttle); } private RetriableStream newThrottledHedgingStream(Throttle throttle) { return new RecordedRetriableStream( method, new Metadata(), channelBufferUsed, PER_RPC_BUFFER_LIMIT, CHANNEL_BUFFER_LIMIT, MoreExecutors.directExecutor(), fakeClock.getScheduledExecutorService(), - RetryPolicy.DEFAULT, HEDGING_POLICY, throttle); + null, HEDGING_POLICY, throttle); } @After @@ -1576,7 +1576,7 @@ public void noRetry_transparentRetry_earlyCommit() { RetriableStream unretriableStream = new RecordedRetriableStream( method, new Metadata(), channelBufferUsed, PER_RPC_BUFFER_LIMIT, CHANNEL_BUFFER_LIMIT, MoreExecutors.directExecutor(), fakeClock.getScheduledExecutorService(), - RetryPolicy.DEFAULT, HedgingPolicy.DEFAULT, null); + null, null, null); // start doReturn(mockStream1).when(retriableStreamRecorder).newSubstream(0); diff --git a/core/src/test/java/io/grpc/internal/RetryPolicyTest.java b/core/src/test/java/io/grpc/internal/RetryPolicyTest.java index 2e02078698a..aab50df7500 100644 --- a/core/src/test/java/io/grpc/internal/RetryPolicyTest.java +++ b/core/src/test/java/io/grpc/internal/RetryPolicyTest.java @@ -77,14 +77,10 @@ public void getRetryPolicies() throws Exception { MethodDescriptor.Builder builder = TestMethodDescriptors.voidMethod().toBuilder(); MethodDescriptor method = builder.setFullMethodName("not/exist").build(); - assertEquals( - RetryPolicy.DEFAULT, - serviceConfigInterceptor.getRetryPolicyFromConfig(method)); + assertThat(serviceConfigInterceptor.getRetryPolicyFromConfig(method)).isNull(); method = builder.setFullMethodName("not_exist/Foo1").build(); - assertEquals( - RetryPolicy.DEFAULT, - serviceConfigInterceptor.getRetryPolicyFromConfig(method)); + assertThat(serviceConfigInterceptor.getRetryPolicyFromConfig(method)).isNull(); method = builder.setFullMethodName("SimpleService1/not_exist").build(); @@ -108,9 +104,7 @@ public void getRetryPolicies() throws Exception { serviceConfigInterceptor.getRetryPolicyFromConfig(method)); method = builder.setFullMethodName("SimpleService2/not_exist").build(); - assertEquals( - RetryPolicy.DEFAULT, - serviceConfigInterceptor.getRetryPolicyFromConfig(method)); + assertThat(serviceConfigInterceptor.getRetryPolicyFromConfig(method)).isNull(); method = builder.setFullMethodName("SimpleService2/Foo2").build(); assertEquals( diff --git a/core/src/test/java/io/grpc/internal/ServiceConfigInterceptorTest.java b/core/src/test/java/io/grpc/internal/ServiceConfigInterceptorTest.java index eaa67480318..4a61587c312 100644 --- a/core/src/test/java/io/grpc/internal/ServiceConfigInterceptorTest.java +++ b/core/src/test/java/io/grpc/internal/ServiceConfigInterceptorTest.java @@ -123,9 +123,9 @@ public void handleUpdateNotCalledBeforeInterceptCall() { verify(channel).newCall(eq(methodDescriptor), callOptionsCap.capture()); assertThat(callOptionsCap.getValue().isWaitForReady()).isFalse(); assertThat(callOptionsCap.getValue().getOption(RETRY_POLICY_KEY).get()) - .isEqualTo(RetryPolicy.DEFAULT); + .isNull(); assertThat(callOptionsCap.getValue().getOption(HEDGING_POLICY_KEY).get()) - .isEqualTo(HedgingPolicy.DEFAULT); + .isNull(); } @Test From 26cd69093be36e2e7a36a56667fdec98d0236b00 Mon Sep 17 00:00:00 2001 From: Benjamin Reed Date: Thu, 27 Aug 2020 10:15:29 -0700 Subject: [PATCH 03/86] benchmarks: Remove -javaagent in CreateStartScripts The Jetty ALPN is not needed for benchmarks. This change removes the -javaagent lines from the benchmarks/build.gradle file as suggested by @ejona86. The shell script references the Jetty ALPN agent stored in ~/.gradle which is not available when just using the application artifacts. interop-testing has a hack to make the agent work, but we don't really care about "plain" Java performance for benchmarks so it is easier to just remove it. --- benchmarks/build.gradle | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle index b653a9d2811..ec071137e0c 100644 --- a/benchmarks/build.gradle +++ b/benchmarks/build.gradle @@ -57,9 +57,7 @@ def vmArgs = [ task qps_client(type: CreateStartScripts) { mainClassName = "io.grpc.benchmarks.qps.AsyncClient" applicationName = "qps_client" - defaultJvmOpts = [ - "-javaagent:" + configurations.alpnagent.asPath - ].plus(vmArgs) + defaultJvmOpts = vmArgs outputDir = new File(project.buildDir, 'tmp/scripts/' + name) classpath = startScripts.classpath } @@ -67,9 +65,7 @@ task qps_client(type: CreateStartScripts) { task openloop_client(type: CreateStartScripts) { mainClassName = "io.grpc.benchmarks.qps.OpenLoopClient" applicationName = "openloop_client" - defaultJvmOpts = [ - "-javaagent:" + configurations.alpnagent.asPath - ].plus(vmArgs) + defaultJvmOpts = vmArgs outputDir = new File(project.buildDir, 'tmp/scripts/' + name) classpath = startScripts.classpath } @@ -84,9 +80,7 @@ task qps_server(type: CreateStartScripts) { task benchmark_worker(type: CreateStartScripts) { mainClassName = "io.grpc.benchmarks.driver.LoadWorker" applicationName = "benchmark_worker" - defaultJvmOpts = [ - "-javaagent:" + configurations.alpnagent.asPath - ].plus(vmArgs) + defaultJvmOpts = vmArgs outputDir = new File(project.buildDir, 'tmp/scripts/' + name) classpath = startScripts.classpath } From 5cfbe1061acefc4a859c9fc85fab8c46f9b7a4b1 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Thu, 27 Aug 2020 14:04:44 -0400 Subject: [PATCH 04/86] api: Fix a small typo in ForwardingChannelBuilderTest --- api/src/test/java/io/grpc/ForwardingChannelBuilderTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/test/java/io/grpc/ForwardingChannelBuilderTest.java b/api/src/test/java/io/grpc/ForwardingChannelBuilderTest.java index a54a687b21e..5ebfef5fb7c 100644 --- a/api/src/test/java/io/grpc/ForwardingChannelBuilderTest.java +++ b/api/src/test/java/io/grpc/ForwardingChannelBuilderTest.java @@ -88,7 +88,7 @@ public void allBuilderMethodsReturnThis() throws Exception { } @Test - public void buildReturnsDelegateBuildByDefualt() { + public void buildReturnsDelegateBuildByDefault() { ManagedChannel mockChannel = mock(ManagedChannel.class); doReturn(mockChannel).when(mockDelegate).build(); From eb6c3415d22950fcac673d76e58dfec426601b80 Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Thu, 27 Aug 2020 16:03:05 -0700 Subject: [PATCH 05/86] xds: fix bug for name resolution error propagation in prioirty_lb --- .../io/grpc/xds/PriorityLoadBalancer.java | 12 ++++-- .../io/grpc/xds/PriorityLoadBalancerTest.java | 37 +++++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/PriorityLoadBalancer.java b/xds/src/main/java/io/grpc/xds/PriorityLoadBalancer.java index 429e2108008..f53ac28b95a 100644 --- a/xds/src/main/java/io/grpc/xds/PriorityLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/PriorityLoadBalancer.java @@ -101,11 +101,15 @@ public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { @Override public void handleNameResolutionError(Status error) { logger.log(XdsLogLevel.WARNING, "Received name resolution error: {0}", error); - if (children.isEmpty()) { - updateOverallState(TRANSIENT_FAILURE, new ErrorPicker(error)); - } + boolean gotoTransientFailure = true; for (ChildLbState child : children.values()) { - child.lb.handleNameResolutionError(error); + if (priorityNames.contains(child.priority)) { + child.lb.handleNameResolutionError(error); + gotoTransientFailure = false; + } + } + if (gotoTransientFailure) { + updateOverallState(TRANSIENT_FAILURE, new ErrorPicker(error)); } } diff --git a/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerTest.java index 1324d6f12e6..95fd1e95220 100644 --- a/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerTest.java @@ -214,6 +214,43 @@ public void handleResolvedAddresses() { verify(barBalancer0, never()).shutdown(); } + @Test + public void handleNameResolutionError() { + Object fooConfig0 = new Object(); + PolicySelection fooPolicy0 = new PolicySelection(fooLbProvider, null, fooConfig0); + Object fooConfig1 = new Object(); + PolicySelection fooPolicy1 = new PolicySelection(fooLbProvider, null, fooConfig1); + + PriorityLbConfig priorityLbConfig = + new PriorityLbConfig(ImmutableMap.of("p0", fooPolicy0), ImmutableList.of("p0")); + priorityLb.handleResolvedAddresses( + ResolvedAddresses.newBuilder() + .setAddresses(ImmutableList.of()) + .setLoadBalancingPolicyConfig(priorityLbConfig) + .build()); + LoadBalancer fooLb0 = Iterables.getOnlyElement(fooBalancers); + Status status = Status.DATA_LOSS.withDescription("fake error"); + priorityLb.handleNameResolutionError(status); + verify(fooLb0).handleNameResolutionError(status); + + priorityLbConfig = + new PriorityLbConfig(ImmutableMap.of("p1", fooPolicy1), ImmutableList.of("p1")); + priorityLb.handleResolvedAddresses( + ResolvedAddresses.newBuilder() + .setAddresses(ImmutableList.of()) + .setLoadBalancingPolicyConfig(priorityLbConfig) + .build()); + assertThat(fooBalancers).hasSize(2); + LoadBalancer fooLb1 = Iterables.getLast(fooBalancers); + status = Status.UNAVAILABLE.withDescription("fake error"); + priorityLb.handleNameResolutionError(status); + // fooLb0 is deactivated but not yet deleted. However, because it is delisted by the latest + // address update, name resolution error will not be propagated to it. + verify(fooLb0, never()).shutdown(); + verify(fooLb0, never()).handleNameResolutionError(status); + verify(fooLb1).handleNameResolutionError(status); + } + @Test public void typicalPriorityFailOverFlow() { PolicySelection policy0 = new PolicySelection(fooLbProvider, null, new Object()); From 09367030ae80b2f5e59e1ab7d4a47642f966d974 Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Fri, 28 Aug 2020 13:00:44 -0700 Subject: [PATCH 06/86] all: fix lint --- compiler/src/java_plugin/cpp/java_generator.h | 4 +- .../ServiceConfigErrorHandlingTest.java | 2 - .../helloworldtls/HelloWorldServerTls.java | 1 - .../InternalProtocolNegotiationEvent.java | 2 + .../main/java/io/grpc/xds/EnvoyProtoData.java | 48 ++++---- .../java/io/grpc/xds/LoadReportClient.java | 8 +- .../java/io/grpc/xds/LoadStatsStoreImpl.java | 4 +- .../io/grpc/xds/PriorityLoadBalancer.java | 1 - .../CertProviderSslContextProvider.java | 2 +- .../rbac/engine/AuthorizationDecision.java | 10 +- .../rbac/engine/AuthorizationEngine.java | 8 +- .../internal/rbac/engine/EvaluateArgs.java | 4 +- .../sds/DynamicSslContextProvider.java | 4 +- .../java/io/grpc/xds/XdsClientImplTest.java | 8 +- .../java/io/grpc/xds/XdsClientImplTestV2.java | 8 +- .../java/io/grpc/xds/XdsClientTestHelper.java | 38 +++--- .../xds/XdsClientWrapperForServerSdsTest.java | 2 +- ...tProviderServerSslContextProviderTest.java | 2 +- ...MeshCaCertificateProviderProviderTest.java | 14 +-- .../engine/AuthzEngineEvaluationTest.java | 108 +++++++++--------- .../internal/rbac/engine/AuthzEngineTest.java | 3 +- .../rbac/engine/EvaluateArgsTest.java | 2 +- .../sds/CommonTlsContextTestsUtil.java | 12 +- 23 files changed, 144 insertions(+), 151 deletions(-) diff --git a/compiler/src/java_plugin/cpp/java_generator.h b/compiler/src/java_plugin/cpp/java_generator.h index 2ecc884efd4..b499c494433 100644 --- a/compiler/src/java_plugin/cpp/java_generator.h +++ b/compiler/src/java_plugin/cpp/java_generator.h @@ -38,13 +38,13 @@ class LogHelper { } }; -// Abort the program after logging the mesage if the given condition is not +// Abort the program after logging the message if the given condition is not // true. Otherwise, do nothing. #define GRPC_CODEGEN_CHECK(x) !(x) && LogHelper(&std::cerr).get_os() \ << "CHECK FAILED: " << __FILE__ << ":" \ << __LINE__ << ": " -// Abort the program after logging the mesage. +// Abort the program after logging the message. #define GRPC_CODEGEN_FAIL GRPC_CODEGEN_CHECK(false) namespace java_grpc_generator { diff --git a/core/src/test/java/io/grpc/internal/ServiceConfigErrorHandlingTest.java b/core/src/test/java/io/grpc/internal/ServiceConfigErrorHandlingTest.java index b8790ad34dc..de6098e5bd7 100644 --- a/core/src/test/java/io/grpc/internal/ServiceConfigErrorHandlingTest.java +++ b/core/src/test/java/io/grpc/internal/ServiceConfigErrorHandlingTest.java @@ -61,7 +61,6 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; import org.mockito.ArgumentCaptor; @@ -103,7 +102,6 @@ public boolean shouldAccept(Runnable command) { private final InternalChannelz channelz = new InternalChannelz(); - @Rule public final ExpectedException thrown = ExpectedException.none(); @Rule public final MockitoRule mocks = MockitoJUnit.rule(); private ManagedChannelImpl channel; diff --git a/examples/example-tls/src/main/java/io/grpc/examples/helloworldtls/HelloWorldServerTls.java b/examples/example-tls/src/main/java/io/grpc/examples/helloworldtls/HelloWorldServerTls.java index 8f5338b038c..4f8a59abc08 100644 --- a/examples/example-tls/src/main/java/io/grpc/examples/helloworldtls/HelloWorldServerTls.java +++ b/examples/example-tls/src/main/java/io/grpc/examples/helloworldtls/HelloWorldServerTls.java @@ -25,7 +25,6 @@ import io.grpc.stub.StreamObserver; import io.netty.handler.ssl.ClientAuth; import io.netty.handler.ssl.SslContextBuilder; -import io.netty.handler.ssl.SslProvider; import java.io.File; import java.io.IOException; diff --git a/netty/src/main/java/io/grpc/netty/InternalProtocolNegotiationEvent.java b/netty/src/main/java/io/grpc/netty/InternalProtocolNegotiationEvent.java index 88cc0ba60aa..3ddeff98b28 100644 --- a/netty/src/main/java/io/grpc/netty/InternalProtocolNegotiationEvent.java +++ b/netty/src/main/java/io/grpc/netty/InternalProtocolNegotiationEvent.java @@ -17,12 +17,14 @@ package io.grpc.netty; import io.grpc.Attributes; +import io.grpc.Internal; import io.grpc.InternalChannelz.Security; import javax.annotation.Nullable; /** * Internal accessor for {@link ProtocolNegotiationEvent}. */ +@Internal public final class InternalProtocolNegotiationEvent { private InternalProtocolNegotiationEvent() {} diff --git a/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java b/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java index dc96aa0f69b..ad78a6967c5 100644 --- a/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java +++ b/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java @@ -1292,8 +1292,8 @@ long getLoadReportIntervalNanos() { io.envoyproxy.envoy.config.endpoint.v3.ClusterStats toEnvoyProtoClusterStats() { io.envoyproxy.envoy.config.endpoint.v3.ClusterStats.Builder builder = - io.envoyproxy.envoy.config.endpoint.v3.ClusterStats.newBuilder(); - builder.setClusterName(clusterName); + io.envoyproxy.envoy.config.endpoint.v3.ClusterStats.newBuilder() + .setClusterName(clusterName); if (clusterServiceName != null) { builder.setClusterServiceName(clusterServiceName); } @@ -1303,15 +1303,16 @@ io.envoyproxy.envoy.config.endpoint.v3.ClusterStats toEnvoyProtoClusterStats() { for (DroppedRequests droppedRequests : droppedRequestsList) { builder.addDroppedRequests(droppedRequests.toEnvoyProtoDroppedRequests()); } - builder.setTotalDroppedRequests(totalDroppedRequests); - builder.setLoadReportInterval(Durations.fromNanos(loadReportIntervalNanos)); - return builder.build(); + return builder + .setTotalDroppedRequests(totalDroppedRequests) + .setLoadReportInterval(Durations.fromNanos(loadReportIntervalNanos)) + .build(); } io.envoyproxy.envoy.api.v2.endpoint.ClusterStats toEnvoyProtoClusterStatsV2() { io.envoyproxy.envoy.api.v2.endpoint.ClusterStats.Builder builder = - io.envoyproxy.envoy.api.v2.endpoint.ClusterStats.newBuilder(); - builder.setClusterName(clusterName); + io.envoyproxy.envoy.api.v2.endpoint.ClusterStats.newBuilder() + .setClusterName(clusterName); for (UpstreamLocalityStats upstreamLocalityStats : upstreamLocalityStatsList) { builder.addUpstreamLocalityStats( upstreamLocalityStats.toEnvoyProtoUpstreamLocalityStatsV2()); @@ -1319,9 +1320,10 @@ io.envoyproxy.envoy.api.v2.endpoint.ClusterStats toEnvoyProtoClusterStatsV2() { for (DroppedRequests droppedRequests : droppedRequestsList) { builder.addDroppedRequests(droppedRequests.toEnvoyProtoDroppedRequestsV2()); } - builder.setTotalDroppedRequests(totalDroppedRequests); - builder.setLoadReportInterval(Durations.fromNanos(loadReportIntervalNanos)); - return builder.build(); + return builder + .setTotalDroppedRequests(totalDroppedRequests) + .setLoadReportInterval(Durations.fromNanos(loadReportIntervalNanos)) + .build(); } @VisibleForTesting @@ -1534,13 +1536,12 @@ List getLoadMetricStatsList() { private io.envoyproxy.envoy.config.endpoint.v3.UpstreamLocalityStats toEnvoyProtoUpstreamLocalityStats() { io.envoyproxy.envoy.config.endpoint.v3.UpstreamLocalityStats.Builder builder - = io.envoyproxy.envoy.config.endpoint.v3.UpstreamLocalityStats.newBuilder(); - builder - .setLocality(locality.toEnvoyProtoLocality()) - .setTotalSuccessfulRequests(totalSuccessfulRequests) - .setTotalErrorRequests(totalErrorRequests) - .setTotalRequestsInProgress(totalRequestsInProgress) - .setTotalIssuedRequests(totalIssuedRequests); + = io.envoyproxy.envoy.config.endpoint.v3.UpstreamLocalityStats.newBuilder() + .setLocality(locality.toEnvoyProtoLocality()) + .setTotalSuccessfulRequests(totalSuccessfulRequests) + .setTotalErrorRequests(totalErrorRequests) + .setTotalRequestsInProgress(totalRequestsInProgress) + .setTotalIssuedRequests(totalIssuedRequests); for (EndpointLoadMetricStats endpointLoadMetricStats : loadMetricStatsList) { builder.addLoadMetricStats(endpointLoadMetricStats.toEnvoyProtoEndpointLoadMetricStats()); } @@ -1550,13 +1551,12 @@ List getLoadMetricStatsList() { private io.envoyproxy.envoy.api.v2.endpoint.UpstreamLocalityStats toEnvoyProtoUpstreamLocalityStatsV2() { io.envoyproxy.envoy.api.v2.endpoint.UpstreamLocalityStats.Builder builder - = io.envoyproxy.envoy.api.v2.endpoint.UpstreamLocalityStats.newBuilder(); - builder - .setLocality(locality.toEnvoyProtoLocalityV2()) - .setTotalSuccessfulRequests(totalSuccessfulRequests) - .setTotalErrorRequests(totalErrorRequests) - .setTotalRequestsInProgress(totalRequestsInProgress) - .setTotalIssuedRequests(totalIssuedRequests); + = io.envoyproxy.envoy.api.v2.endpoint.UpstreamLocalityStats.newBuilder() + .setLocality(locality.toEnvoyProtoLocalityV2()) + .setTotalSuccessfulRequests(totalSuccessfulRequests) + .setTotalErrorRequests(totalErrorRequests) + .setTotalRequestsInProgress(totalRequestsInProgress) + .setTotalIssuedRequests(totalIssuedRequests); for (EndpointLoadMetricStats endpointLoadMetricStats : loadMetricStatsList) { builder.addLoadMetricStats(endpointLoadMetricStats.toEnvoyProtoEndpointLoadMetricStatsV2()); } diff --git a/xds/src/main/java/io/grpc/xds/LoadReportClient.java b/xds/src/main/java/io/grpc/xds/LoadReportClient.java index c9cff2db6c4..119ee4dfdcc 100644 --- a/xds/src/main/java/io/grpc/xds/LoadReportClient.java +++ b/xds/src/main/java/io/grpc/xds/LoadReportClient.java @@ -416,8 +416,8 @@ private static final class LoadStatsRequestData { io.envoyproxy.envoy.service.load_stats.v2.LoadStatsRequest toEnvoyProtoV2() { io.envoyproxy.envoy.service.load_stats.v2.LoadStatsRequest.Builder builder - = io.envoyproxy.envoy.service.load_stats.v2.LoadStatsRequest.newBuilder(); - builder.setNode(node.toEnvoyProtoNodeV2()); + = io.envoyproxy.envoy.service.load_stats.v2.LoadStatsRequest.newBuilder() + .setNode(node.toEnvoyProtoNodeV2()); if (clusterStatsList != null) { for (ClusterStats stats : clusterStatsList) { builder.addClusterStats(stats.toEnvoyProtoClusterStatsV2()); @@ -427,8 +427,8 @@ io.envoyproxy.envoy.service.load_stats.v2.LoadStatsRequest toEnvoyProtoV2() { } LoadStatsRequest toEnvoyProtoV3() { - LoadStatsRequest.Builder builder = LoadStatsRequest.newBuilder(); - builder.setNode(node.toEnvoyProtoNode()); + LoadStatsRequest.Builder builder = LoadStatsRequest.newBuilder() + .setNode(node.toEnvoyProtoNode()); if (clusterStatsList != null) { for (ClusterStats stats : clusterStatsList) { builder.addClusterStats(stats.toEnvoyProtoClusterStats()); diff --git a/xds/src/main/java/io/grpc/xds/LoadStatsStoreImpl.java b/xds/src/main/java/io/grpc/xds/LoadStatsStoreImpl.java index afb91c3ecc7..d4654987252 100644 --- a/xds/src/main/java/io/grpc/xds/LoadStatsStoreImpl.java +++ b/xds/src/main/java/io/grpc/xds/LoadStatsStoreImpl.java @@ -17,6 +17,7 @@ package io.grpc.xds; import static com.google.common.base.Preconditions.checkNotNull; +import static java.util.concurrent.TimeUnit.NANOSECONDS; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Stopwatch; @@ -33,7 +34,6 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import javax.annotation.Nullable; import javax.annotation.concurrent.NotThreadSafe; @@ -113,7 +113,7 @@ public ClusterStats generateLoadReport() { statsBuilder.addDroppedRequests(new DroppedRequests(entry.getKey(),drops)); } statsBuilder.setTotalDroppedRequests(totalDrops); - statsBuilder.setLoadReportIntervalNanos(stopwatch.elapsed(TimeUnit.NANOSECONDS)); + statsBuilder.setLoadReportIntervalNanos(stopwatch.elapsed(NANOSECONDS)); stopwatch.reset().start(); return statsBuilder.build(); } diff --git a/xds/src/main/java/io/grpc/xds/PriorityLoadBalancer.java b/xds/src/main/java/io/grpc/xds/PriorityLoadBalancer.java index f53ac28b95a..d5876fd7432 100644 --- a/xds/src/main/java/io/grpc/xds/PriorityLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/PriorityLoadBalancer.java @@ -260,7 +260,6 @@ public void run() { policy = newPolicy; lb.switchTo(lbProvider); } - // TODO(zdapeng): Implement address filtering. lb.handleResolvedAddresses( addresses .toBuilder() diff --git a/xds/src/main/java/io/grpc/xds/internal/certprovider/CertProviderSslContextProvider.java b/xds/src/main/java/io/grpc/xds/internal/certprovider/CertProviderSslContextProvider.java index 5f03e3becca..eef5ee551e7 100644 --- a/xds/src/main/java/io/grpc/xds/internal/certprovider/CertProviderSslContextProvider.java +++ b/xds/src/main/java/io/grpc/xds/internal/certprovider/CertProviderSslContextProvider.java @@ -83,7 +83,7 @@ protected CertProviderSslContextProvider( } } - private CertificateProviderInfo getCertProviderConfig( + private static CertificateProviderInfo getCertProviderConfig( Map certProviders, String pluginInstanceName) { return certProviders.get(pluginInstanceName); } diff --git a/xds/src/main/java/io/grpc/xds/internal/rbac/engine/AuthorizationDecision.java b/xds/src/main/java/io/grpc/xds/internal/rbac/engine/AuthorizationDecision.java index 474ff7c180f..43f69dc6bf1 100644 --- a/xds/src/main/java/io/grpc/xds/internal/rbac/engine/AuthorizationDecision.java +++ b/xds/src/main/java/io/grpc/xds/internal/rbac/engine/AuthorizationDecision.java @@ -71,20 +71,20 @@ public ImmutableList getPolicyNames() { public String toString() { StringBuilder authzStr = new StringBuilder(); switch (this.decision) { - case ALLOW: + case ALLOW: authzStr.append("Authorization Decision: ALLOW. \n"); break; - case DENY: + case DENY: authzStr.append("Authorization Decision: DENY. \n"); break; - case UNKNOWN: + case UNKNOWN: authzStr.append("Authorization Decision: UNKNOWN. \n"); break; - default: + default: break; } for (String policyName : this.policyNames) { - authzStr.append(policyName + "; \n"); + authzStr.append(policyName).append("; \n"); } return authzStr.toString(); } diff --git a/xds/src/main/java/io/grpc/xds/internal/rbac/engine/AuthorizationEngine.java b/xds/src/main/java/io/grpc/xds/internal/rbac/engine/AuthorizationEngine.java index bb3005883f4..a7e09b0f3fc 100644 --- a/xds/src/main/java/io/grpc/xds/internal/rbac/engine/AuthorizationEngine.java +++ b/xds/src/main/java/io/grpc/xds/internal/rbac/engine/AuthorizationEngine.java @@ -102,7 +102,7 @@ public AuthorizationEngine(RBAC rbacPolicy) { * @param allowPolicy input Envoy RBAC policy with ALLOW action. * @throws IllegalArgumentException if the user inputs an invalid RBAC list. */ - public AuthorizationEngine(RBAC denyPolicy, RBAC allowPolicy) throws IllegalArgumentException { + public AuthorizationEngine(RBAC denyPolicy, RBAC allowPolicy) { checkArgument( denyPolicy.getAction() == Action.DENY && allowPolicy.getAction() == Action.ALLOW, "Invalid RBAC list, " @@ -140,7 +140,7 @@ public AuthorizationDecision evaluate(EvaluateArgs args) { if (authzDecision != null) { return authzDecision; } - if (unknownPolicyNames.size() > 0) { + if (!unknownPolicyNames.isEmpty()) { return new AuthorizationDecision( AuthorizationDecision.Output.UNKNOWN, unknownPolicyNames); } @@ -154,7 +154,7 @@ public AuthorizationDecision evaluate(EvaluateArgs args) { if (authzDecision != null) { return authzDecision; } - if (unknownPolicyNames.size() > 0) { + if (!unknownPolicyNames.isEmpty()) { return new AuthorizationDecision( AuthorizationDecision.Output.UNKNOWN, unknownPolicyNames); } @@ -198,7 +198,7 @@ protected boolean matches(Expr condition, Activation activation) throws Interpre try { Object result = interpretable.eval(activation); if (result instanceof Boolean) { - return Boolean.valueOf(result.toString()); + return Boolean.parseBoolean(result.toString()); } // Throw an InterpreterException if there are missing Envoy Attributes. if (result instanceof IncompleteData) { diff --git a/xds/src/main/java/io/grpc/xds/internal/rbac/engine/EvaluateArgs.java b/xds/src/main/java/io/grpc/xds/internal/rbac/engine/EvaluateArgs.java index e8ed1828a22..35945a21026 100644 --- a/xds/src/main/java/io/grpc/xds/internal/rbac/engine/EvaluateArgs.java +++ b/xds/src/main/java/io/grpc/xds/internal/rbac/engine/EvaluateArgs.java @@ -23,8 +23,8 @@ /** The EvaluateArgs class holds evaluate arguments used in CEL-based Authorization Engine. */ public class EvaluateArgs { - private Metadata headers; - private ServerCall call; + private final Metadata headers; + private final ServerCall call; /** * Creates a new EvaluateArgs using the input {@code headers} for resolving headers diff --git a/xds/src/main/java/io/grpc/xds/internal/sds/DynamicSslContextProvider.java b/xds/src/main/java/io/grpc/xds/internal/sds/DynamicSslContextProvider.java index 7f40b822f01..c75347c1f5e 100644 --- a/xds/src/main/java/io/grpc/xds/internal/sds/DynamicSslContextProvider.java +++ b/xds/src/main/java/io/grpc/xds/internal/sds/DynamicSslContextProvider.java @@ -75,8 +75,8 @@ protected final void updateSslContext() { alpnList); sslContextBuilder.applicationProtocolConfig(apn); } - List pendingCallbacksCopy = null; - SslContext sslContextCopy = null; + List pendingCallbacksCopy; + SslContext sslContextCopy; synchronized (pendingCallbacks) { sslContext = sslContextBuilder.build(); sslContextCopy = sslContext; diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java index c06a6cc5fc4..b55a649741c 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java @@ -749,7 +749,7 @@ public void resolveVirtualHostWithPathMatchingInRdsResponse() { new EnvoyProtoData.Route( // path match with cluster route new io.grpc.xds.RouteMatch( - /* pathPrefixMatch= */ null,/* pathExactMatch= */ "/service1/method1"), + /* pathPrefixMatch= */ null, /* pathExactMatch= */ "/service1/method1"), new EnvoyProtoData.RouteAction( TimeUnit.SECONDS.toNanos(15L), "cl1.googleapis.com", null))); assertThat(routes.get(1)) @@ -757,7 +757,7 @@ public void resolveVirtualHostWithPathMatchingInRdsResponse() { new EnvoyProtoData.Route( // path match with weighted cluster route new io.grpc.xds.RouteMatch( - /* pathPrefixMatch= */ null,/* pathExactMatch= */ "/service2/method2"), + /* pathPrefixMatch= */ null, /* pathExactMatch= */ "/service2/method2"), new EnvoyProtoData.RouteAction( TimeUnit.SECONDS.toNanos(15L), null, @@ -769,7 +769,7 @@ public void resolveVirtualHostWithPathMatchingInRdsResponse() { new EnvoyProtoData.Route( // prefix match with cluster route new io.grpc.xds.RouteMatch( - /* pathPrefixMatch= */ "/service1/",/* pathExactMatch= */ null), + /* pathPrefixMatch= */ "/service1/", /* pathExactMatch= */ null), new EnvoyProtoData.RouteAction( TimeUnit.SECONDS.toNanos(15L), "cl1.googleapis.com", null))); assertThat(routes.get(3)) @@ -777,7 +777,7 @@ public void resolveVirtualHostWithPathMatchingInRdsResponse() { new EnvoyProtoData.Route( // default match with cluster route new io.grpc.xds.RouteMatch( - /* pathPrefixMatch= */ "",/* pathExactMatch= */ null), + /* pathPrefixMatch= */ "", /* pathExactMatch= */ null), new EnvoyProtoData.RouteAction( TimeUnit.SECONDS.toNanos(15L), "cluster.googleapis.com", null))); } diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java index 64da95d25b7..d3691a9a475 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java @@ -751,7 +751,7 @@ public void resolveVirtualHostWithPathMatchingInRdsResponse() { new EnvoyProtoData.Route( // path match with cluster route new io.grpc.xds.RouteMatch( - /* pathPrefixMatch= */ null,/* pathExactMatch= */ "/service1/method1"), + /* pathPrefixMatch= */ null, /* pathExactMatch= */ "/service1/method1"), new EnvoyProtoData.RouteAction( TimeUnit.SECONDS.toNanos(15L), "cl1.googleapis.com", null))); assertThat(routes.get(1)) @@ -759,7 +759,7 @@ public void resolveVirtualHostWithPathMatchingInRdsResponse() { new EnvoyProtoData.Route( // path match with weighted cluster route new io.grpc.xds.RouteMatch( - /* pathPrefixMatch= */ null,/* pathExactMatch= */ "/service2/method2"), + /* pathPrefixMatch= */ null, /* pathExactMatch= */ "/service2/method2"), new EnvoyProtoData.RouteAction( TimeUnit.SECONDS.toNanos(15L), null, @@ -771,7 +771,7 @@ public void resolveVirtualHostWithPathMatchingInRdsResponse() { new EnvoyProtoData.Route( // prefix match with cluster route new io.grpc.xds.RouteMatch( - /* pathPrefixMatch= */ "/service1/",/* pathExactMatch= */ null), + /* pathPrefixMatch= */ "/service1/", /* pathExactMatch= */ null), new EnvoyProtoData.RouteAction( TimeUnit.SECONDS.toNanos(15L), "cl1.googleapis.com", null))); assertThat(routes.get(3)) @@ -779,7 +779,7 @@ public void resolveVirtualHostWithPathMatchingInRdsResponse() { new EnvoyProtoData.Route( // default match with cluster route new io.grpc.xds.RouteMatch( - /* pathPrefixMatch= */ "",/* pathExactMatch= */ null), + /* pathPrefixMatch= */ "", /* pathExactMatch= */ null), new EnvoyProtoData.RouteAction( TimeUnit.SECONDS.toNanos(15L), "cluster.googleapis.com", null))); } diff --git a/xds/src/test/java/io/grpc/xds/XdsClientTestHelper.java b/xds/src/test/java/io/grpc/xds/XdsClientTestHelper.java index 19bbed263b5..3220a77e524 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientTestHelper.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientTestHelper.java @@ -22,6 +22,7 @@ import io.envoyproxy.envoy.config.cluster.v3.Cluster; import io.envoyproxy.envoy.config.cluster.v3.Cluster.DiscoveryType; import io.envoyproxy.envoy.config.cluster.v3.Cluster.EdsClusterConfig; +import io.envoyproxy.envoy.config.cluster.v3.Cluster.LbPolicy; import io.envoyproxy.envoy.config.core.v3.Address; import io.envoyproxy.envoy.config.core.v3.AggregatedConfigSource; import io.envoyproxy.envoy.config.core.v3.ApiConfigSource; @@ -34,6 +35,8 @@ import io.envoyproxy.envoy.config.core.v3.SocketAddress; import io.envoyproxy.envoy.config.core.v3.TransportSocket; import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment; +import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment.Policy; +import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment.Policy.DropOverload; import io.envoyproxy.envoy.config.endpoint.v3.Endpoint; import io.envoyproxy.envoy.config.endpoint.v3.LbEndpoint; import io.envoyproxy.envoy.config.endpoint.v3.LocalityLbEndpoints; @@ -206,7 +209,7 @@ static Cluster buildSecureCluster( edsClusterConfigBuilder.setServiceName(edsServiceName); } clusterBuilder.setEdsClusterConfig(edsClusterConfigBuilder); - clusterBuilder.setLbPolicy(Cluster.LbPolicy.ROUND_ROBIN); + clusterBuilder.setLbPolicy(LbPolicy.ROUND_ROBIN); if (enableLrs) { clusterBuilder.setLrsServer( ConfigSource.newBuilder() @@ -223,19 +226,20 @@ static io.envoyproxy.envoy.api.v2.Cluster buildSecureClusterV2( String clusterName, @Nullable String edsServiceName, boolean enableLrs, @Nullable io.envoyproxy.envoy.api.v2.auth.UpstreamTlsContext upstreamTlsContext) { io.envoyproxy.envoy.api.v2.Cluster.Builder clusterBuilder = - io.envoyproxy.envoy.api.v2.Cluster.newBuilder(); - clusterBuilder.setName(clusterName); - clusterBuilder.setType(io.envoyproxy.envoy.api.v2.Cluster.DiscoveryType.EDS); + io.envoyproxy.envoy.api.v2.Cluster.newBuilder() + .setName(clusterName) + .setType(io.envoyproxy.envoy.api.v2.Cluster.DiscoveryType.EDS); io.envoyproxy.envoy.api.v2.Cluster.EdsClusterConfig.Builder edsClusterConfigBuilder = - io.envoyproxy.envoy.api.v2.Cluster.EdsClusterConfig.newBuilder(); - edsClusterConfigBuilder.setEdsConfig( - io.envoyproxy.envoy.api.v2.core.ConfigSource.newBuilder() - .setAds(io.envoyproxy.envoy.api.v2.core.AggregatedConfigSource.getDefaultInstance())); + io.envoyproxy.envoy.api.v2.Cluster.EdsClusterConfig.newBuilder() + .setEdsConfig( + io.envoyproxy.envoy.api.v2.core.ConfigSource.newBuilder().setAds( + io.envoyproxy.envoy.api.v2.core.AggregatedConfigSource.getDefaultInstance())); if (edsServiceName != null) { edsClusterConfigBuilder.setServiceName(edsServiceName); } - clusterBuilder.setEdsClusterConfig(edsClusterConfigBuilder); - clusterBuilder.setLbPolicy(io.envoyproxy.envoy.api.v2.Cluster.LbPolicy.ROUND_ROBIN); + clusterBuilder + .setEdsClusterConfig(edsClusterConfigBuilder) + .setLbPolicy(io.envoyproxy.envoy.api.v2.Cluster.LbPolicy.ROUND_ROBIN); if (enableLrs) { clusterBuilder.setLrsServer( io.envoyproxy.envoy.api.v2.core.ConfigSource.newBuilder() @@ -250,19 +254,16 @@ static io.envoyproxy.envoy.api.v2.Cluster buildSecureClusterV2( } static ClusterLoadAssignment buildClusterLoadAssignment(String clusterName, - List localityLbEndpoints, - List dropOverloads) { + List localityLbEndpoints, List dropOverloads) { return ClusterLoadAssignment.newBuilder() .setClusterName(clusterName) .addAllEndpoints(localityLbEndpoints) - .setPolicy( - ClusterLoadAssignment.Policy.newBuilder() - .addAllDropOverloads(dropOverloads)) + .setPolicy(Policy.newBuilder().addAllDropOverloads(dropOverloads)) .build(); } - @SuppressWarnings("deprecation") + @SuppressWarnings("deprecation") // disableOverprovisioning is deprecated by needed for v2 static io.envoyproxy.envoy.api.v2.ClusterLoadAssignment buildClusterLoadAssignmentV2( String clusterName, List localityLbEndpoints, @@ -278,10 +279,9 @@ static io.envoyproxy.envoy.api.v2.ClusterLoadAssignment buildClusterLoadAssignme .build(); } - static ClusterLoadAssignment.Policy.DropOverload buildDropOverload( - String category, int dropPerMillion) { + static DropOverload buildDropOverload(String category, int dropPerMillion) { return - ClusterLoadAssignment.Policy.DropOverload.newBuilder() + DropOverload.newBuilder() .setCategory(category) .setDropPercentage( FractionalPercent.newBuilder() diff --git a/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTest.java b/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTest.java index 7965eeec368..7e099de6c34 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTest.java @@ -109,7 +109,7 @@ public static Iterable data() { "exact IP over IPANY match, expect filter2" }, { - PORT,// matches dest port but no address match + PORT, // matches dest port but no address match "168.20.20.2", "10.1.2.4", "192.168.10.1", diff --git a/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderServerSslContextProviderTest.java b/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderServerSslContextProviderTest.java index 3475c6e151a..9de2081ed7c 100644 --- a/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderServerSslContextProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderServerSslContextProviderTest.java @@ -17,7 +17,6 @@ package io.grpc.xds.internal.certprovider; import static com.google.common.truth.Truth.assertThat; -import static io.grpc.xds.internal.certprovider.CertProviderClientSslContextProviderTest.QueuedExecutor; import static io.grpc.xds.internal.certprovider.CommonCertProviderTestUtils.getCertFromResourceName; import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.CA_PEM_FILE; import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.CLIENT_PEM_FILE; @@ -34,6 +33,7 @@ import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CertificateValidationContext; import io.grpc.xds.Bootstrapper; import io.grpc.xds.EnvoyServerProtoData; +import io.grpc.xds.internal.certprovider.CertProviderClientSslContextProviderTest.QueuedExecutor; import io.grpc.xds.internal.sds.CommonTlsContextTestsUtil; import io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.TestCallback; import org.junit.Before; diff --git a/xds/src/test/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderProviderTest.java b/xds/src/test/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderProviderTest.java index 5da5b34d030..123db14a059 100644 --- a/xds/src/test/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderProviderTest.java @@ -226,32 +226,32 @@ public void createProvider_nonDefaultFullConfig() throws IOException { eq(TimeUnit.SECONDS.toMillis(RPC_TIMEOUT_SECONDS))); } - private Map buildFullConfig() throws IOException { + private static Map buildFullConfig() throws IOException { return getCertProviderConfig(CommonCertProviderTestUtils.getNonDefaultTestBootstrapInfo()); } - private Map buildMinimalConfig() throws IOException { + private static Map buildMinimalConfig() throws IOException { return getCertProviderConfig(CommonCertProviderTestUtils.getMinimalBootstrapInfo()); } - private Map buildBadClusterUrlConfig() throws IOException { + private static Map buildBadClusterUrlConfig() throws IOException { return getCertProviderConfig( CommonCertProviderTestUtils.getMinimalAndBadClusterUrlBootstrapInfo()); } - private Map buildMissingSaJwtLocationConfig() throws IOException { + private static Map buildMissingSaJwtLocationConfig() throws IOException { return getCertProviderConfig(CommonCertProviderTestUtils.getMissingSaJwtLocation()); } - private Map buildMissingGkeClusterUrlConfig() throws IOException { + private static Map buildMissingGkeClusterUrlConfig() throws IOException { return getCertProviderConfig(CommonCertProviderTestUtils.getMissingGkeClusterUrl()); } - private Map buildBadChannelCredsConfig() throws IOException { + private static Map buildBadChannelCredsConfig() throws IOException { return getCertProviderConfig(CommonCertProviderTestUtils.getBadChannelCredsConfig()); } - private Map getCertProviderConfig(Bootstrapper.BootstrapInfo bootstrapInfo) { + private static Map getCertProviderConfig(Bootstrapper.BootstrapInfo bootstrapInfo) { Map certProviders = bootstrapInfo.getCertProviders(); Bootstrapper.CertificateProviderInfo gcpIdInfo = diff --git a/xds/src/test/java/io/grpc/xds/internal/rbac/engine/AuthzEngineEvaluationTest.java b/xds/src/test/java/io/grpc/xds/internal/rbac/engine/AuthzEngineEvaluationTest.java index 61172d576c0..bd8d35744b6 100644 --- a/xds/src/test/java/io/grpc/xds/internal/rbac/engine/AuthzEngineEvaluationTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/rbac/engine/AuthzEngineEvaluationTest.java @@ -16,8 +16,8 @@ package io.grpc.xds.internal.rbac.engine; +import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; @@ -31,7 +31,6 @@ import io.envoyproxy.envoy.config.rbac.v2.RBAC.Action; import io.grpc.xds.internal.rbac.engine.cel.Activation; import io.grpc.xds.internal.rbac.engine.cel.InterpreterException; -import java.lang.StringBuilder; import java.util.Map; import org.junit.Before; import org.junit.Rule; @@ -51,14 +50,9 @@ public class AuthzEngineEvaluationTest { @Mock private EvaluateArgs args; - - @Mock - private Activation activation; - @Mock private Map attributes; - private AuthorizationEngine engine; private AuthorizationEngine spyEngine; private AuthorizationDecision evaluateResult; @@ -130,7 +124,7 @@ public void buildRbac() { @Before public void setupEngineSingleRbacAllow() { buildRbac(); - engine = new AuthorizationEngine(rbacAllow); + AuthorizationEngine engine = new AuthorizationEngine(rbacAllow); spyEngine = Mockito.spy(engine); doReturn(ImmutableMap.copyOf(attributes)).when(args).generateEnvoyAttributes(); } @@ -139,7 +133,7 @@ public void setupEngineSingleRbacAllow() { @Before public void setupEngineSingleRbacDeny() { buildRbac(); - engine = new AuthorizationEngine(rbacDeny); + AuthorizationEngine engine = new AuthorizationEngine(rbacDeny); spyEngine = Mockito.spy(engine); doReturn(ImmutableMap.copyOf(attributes)).when(args).generateEnvoyAttributes(); } @@ -148,7 +142,7 @@ public void setupEngineSingleRbacDeny() { @Before public void setupEngineRbacPair() { buildRbac(); - engine = new AuthorizationEngine(rbacDeny, rbacAllow); + AuthorizationEngine engine = new AuthorizationEngine(rbacDeny, rbacAllow); spyEngine = Mockito.spy(engine); doReturn(ImmutableMap.copyOf(attributes)).when(args).generateEnvoyAttributes(); } @@ -166,9 +160,9 @@ public void testAllowEngineWithAllMatchedPolicies() throws InterpreterException doReturn(true).when(spyEngine).matches(eq(condition2), any(Activation.class)); doReturn(true).when(spyEngine).matches(eq(condition3), any(Activation.class)); evaluateResult = spyEngine.evaluate(args); - assertEquals(evaluateResult.getDecision(), AuthorizationDecision.Output.ALLOW); - assertEquals(evaluateResult.getPolicyNames().size(), 1); - assertTrue(evaluateResult.getPolicyNames().contains("Policy 1")); + assertThat(evaluateResult.getDecision()).isEqualTo(AuthorizationDecision.Output.ALLOW); + assertThat(evaluateResult.getPolicyNames()).hasSize(1); + assertThat(evaluateResult.getPolicyNames()).contains("Policy 1"); } /** @@ -184,9 +178,9 @@ public void testAllowEngineWithAllUnmatchedPolicies() throws InterpreterExceptio doReturn(false).when(spyEngine).matches(eq(condition2), any(Activation.class)); doReturn(false).when(spyEngine).matches(eq(condition3), any(Activation.class)); evaluateResult = spyEngine.evaluate(args); - assertEquals(evaluateResult.getDecision(), AuthorizationDecision.Output.DENY); - assertEquals(evaluateResult.getPolicyNames().size(), 0); - assertEquals(evaluateResult.toString(), + assertThat(evaluateResult.getDecision()).isEqualTo(AuthorizationDecision.Output.DENY); + assertThat(evaluateResult.getPolicyNames()).isEmpty(); + assertThat(evaluateResult.toString()).isEqualTo( new StringBuilder("Authorization Decision: DENY. \n").toString()); } @@ -207,7 +201,7 @@ public void testAllowEngineWithMatchedAndUnmatchedPolicies() evaluateResult = spyEngine.evaluate(args); assertEquals(evaluateResult.getDecision(), AuthorizationDecision.Output.ALLOW); assertEquals(evaluateResult.getPolicyNames().size(), 1); - assertTrue(evaluateResult.getPolicyNames().contains("Policy 2")); + assertThat(evaluateResult.getPolicyNames()).contains("Policy 2"); } /** @@ -226,11 +220,11 @@ public void testAllowEngineWithUnknownAndUnmatchedPolicies() doThrow(new InterpreterException.Builder("Unknown result").build()) .when(spyEngine).matches(eq(condition3), any(Activation.class)); evaluateResult = spyEngine.evaluate(args); - assertEquals(evaluateResult.getDecision(), AuthorizationDecision.Output.UNKNOWN); - assertEquals(evaluateResult.getPolicyNames().size(), 2); - assertTrue(evaluateResult.getPolicyNames().contains("Policy 2")); - assertTrue(evaluateResult.getPolicyNames().contains("Policy 3")); - assertEquals(evaluateResult.toString(), + assertThat(evaluateResult.getDecision()).isEqualTo(AuthorizationDecision.Output.UNKNOWN); + assertThat(evaluateResult.getPolicyNames()).hasSize(2); + assertThat(evaluateResult.getPolicyNames()).contains("Policy 2"); + assertThat(evaluateResult.getPolicyNames()).contains("Policy 3"); + assertThat(evaluateResult.toString()).isEqualTo( new StringBuilder("Authorization Decision: UNKNOWN. \n" + "Policy 2; \n" + "Policy 3; \n").toString()); } @@ -250,10 +244,10 @@ public void testAllowEngineWithMatchedUnmatchedAndUnknownPolicies() doThrow(new InterpreterException.Builder("Unknown result").build()) .when(spyEngine).matches(eq(condition3), any(Activation.class)); evaluateResult = spyEngine.evaluate(args); - assertEquals(evaluateResult.getDecision(), AuthorizationDecision.Output.ALLOW); - assertEquals(evaluateResult.getPolicyNames().size(), 1); - assertTrue(evaluateResult.getPolicyNames().contains("Policy 2")); - assertEquals(evaluateResult.toString(), + assertThat(evaluateResult.getDecision()).isEqualTo(AuthorizationDecision.Output.ALLOW); + assertThat(evaluateResult.getPolicyNames()).hasSize(1); + assertThat(evaluateResult.getPolicyNames()).contains("Policy 2"); + assertThat(evaluateResult.toString()).isEqualTo( new StringBuilder("Authorization Decision: ALLOW. \n" + "Policy 2; \n").toString()); } @@ -270,9 +264,9 @@ public void testDenyEngineWithAllMatchedPolicies() throws InterpreterException { doReturn(true).when(spyEngine).matches(eq(condition5), any(Activation.class)); doReturn(true).when(spyEngine).matches(eq(condition6), any(Activation.class)); evaluateResult = spyEngine.evaluate(args); - assertEquals(evaluateResult.getDecision(), AuthorizationDecision.Output.DENY); - assertEquals(evaluateResult.getPolicyNames().size(), 1); - assertTrue(evaluateResult.getPolicyNames().contains("Policy 4")); + assertThat(evaluateResult.getDecision()).isEqualTo(AuthorizationDecision.Output.DENY); + assertThat(evaluateResult.getPolicyNames()).hasSize(1); + assertThat(evaluateResult.getPolicyNames()).contains("Policy 4"); } /** @@ -288,8 +282,8 @@ public void testDenyEngineWithAllUnmatchedPolicies() throws InterpreterException doReturn(false).when(spyEngine).matches(eq(condition5), any(Activation.class)); doReturn(false).when(spyEngine).matches(eq(condition6), any(Activation.class)); evaluateResult = spyEngine.evaluate(args); - assertEquals(evaluateResult.getDecision(), AuthorizationDecision.Output.ALLOW); - assertEquals(evaluateResult.getPolicyNames().size(), 0); + assertThat(evaluateResult.getDecision()).isEqualTo(AuthorizationDecision.Output.ALLOW); + assertThat(evaluateResult.getPolicyNames()).isEmpty(); } /** @@ -307,9 +301,9 @@ public void testDenyEngineWithMatchedAndUnmatchedPolicies() doReturn(true).when(spyEngine).matches(eq(condition5), any(Activation.class)); doReturn(true).when(spyEngine).matches(eq(condition6), any(Activation.class)); evaluateResult = spyEngine.evaluate(args); - assertEquals(evaluateResult.getDecision(), AuthorizationDecision.Output.DENY); - assertEquals(evaluateResult.getPolicyNames().size(), 1); - assertTrue(evaluateResult.getPolicyNames().contains("Policy 5")); + assertThat(evaluateResult.getDecision()).isEqualTo(AuthorizationDecision.Output.DENY); + assertThat(evaluateResult.getPolicyNames()).hasSize(1); + assertThat(evaluateResult.getPolicyNames()).contains("Policy 5"); } /** @@ -328,10 +322,10 @@ public void testDenyEngineWithUnknownAndUnmatchedPolicies() doThrow(new InterpreterException.Builder("Unknown result").build()) .when(spyEngine).matches(eq(condition6), any(Activation.class)); evaluateResult = spyEngine.evaluate(args); - assertEquals(evaluateResult.getDecision(), AuthorizationDecision.Output.UNKNOWN); - assertEquals(evaluateResult.getPolicyNames().size(), 2); - assertTrue(evaluateResult.getPolicyNames().contains("Policy 5")); - assertTrue(evaluateResult.getPolicyNames().contains("Policy 6")); + assertThat(evaluateResult.getDecision()).isEqualTo(AuthorizationDecision.Output.UNKNOWN); + assertThat(evaluateResult.getPolicyNames()).hasSize(2); + assertThat(evaluateResult.getPolicyNames()).contains("Policy 5"); + assertThat(evaluateResult.getPolicyNames()).contains("Policy 6"); } /** @@ -349,9 +343,9 @@ public void testDenyEngineWithMatchedUnmatchedAndUnknownPolicies() doThrow(new InterpreterException.Builder("Unknown result").build()) .when(spyEngine).matches(eq(condition6), any(Activation.class)); evaluateResult = spyEngine.evaluate(args); - assertEquals(evaluateResult.getDecision(), AuthorizationDecision.Output.DENY); - assertEquals(evaluateResult.getPolicyNames().size(), 1); - assertTrue(evaluateResult.getPolicyNames().contains("Policy 5")); + assertThat(evaluateResult.getDecision()).isEqualTo(AuthorizationDecision.Output.DENY); + assertThat(evaluateResult.getPolicyNames()).hasSize(1); + assertThat(evaluateResult.getPolicyNames()).contains("Policy 5"); } /** @@ -371,9 +365,9 @@ public void testEnginePairWithAllMatchedDenyEngine() throws InterpreterException doReturn(true).when(spyEngine).matches(eq(condition5), any(Activation.class)); doReturn(true).when(spyEngine).matches(eq(condition6), any(Activation.class)); evaluateResult = spyEngine.evaluate(args); - assertEquals(evaluateResult.getDecision(), AuthorizationDecision.Output.DENY); - assertEquals(evaluateResult.getPolicyNames().size(), 1); - assertTrue(evaluateResult.getPolicyNames().contains("Policy 4")); + assertThat(evaluateResult.getDecision()).isEqualTo(AuthorizationDecision.Output.DENY); + assertThat(evaluateResult.getPolicyNames()).hasSize(1); + assertThat(evaluateResult.getPolicyNames()).contains("Policy 4"); } /** @@ -396,9 +390,9 @@ public void testEnginePairWithPartiallyMatchedDenyEngine() doThrow(new InterpreterException.Builder("Unknown result").build()) .when(spyEngine).matches(eq(condition6), any(Activation.class)); evaluateResult = spyEngine.evaluate(args); - assertEquals(evaluateResult.getDecision(), AuthorizationDecision.Output.DENY); - assertEquals(evaluateResult.getPolicyNames().size(), 1); - assertTrue(evaluateResult.getPolicyNames().contains("Policy 5")); + assertThat(evaluateResult.getDecision()).isEqualTo(AuthorizationDecision.Output.DENY); + assertThat(evaluateResult.getPolicyNames()).hasSize(1); + assertThat(evaluateResult.getPolicyNames()).contains("Policy 5"); } /** @@ -419,10 +413,10 @@ public void testEnginePairWithUnknownDenyEngine() throws InterpreterException { doThrow(new InterpreterException.Builder("Unknown result").build()) .when(spyEngine).matches(eq(condition6), any(Activation.class)); evaluateResult = spyEngine.evaluate(args); - assertEquals(evaluateResult.getDecision(), AuthorizationDecision.Output.UNKNOWN); - assertEquals(evaluateResult.getPolicyNames().size(), 2); - assertTrue(evaluateResult.getPolicyNames().contains("Policy 5")); - assertTrue(evaluateResult.getPolicyNames().contains("Policy 6")); + assertThat(evaluateResult.getDecision()).isEqualTo(AuthorizationDecision.Output.UNKNOWN); + assertThat(evaluateResult.getPolicyNames()).hasSize(2); + assertThat(evaluateResult.getPolicyNames()).contains("Policy 5"); + assertThat(evaluateResult.getPolicyNames()).contains("Policy 6"); } /** @@ -446,10 +440,10 @@ public void testEnginePairWithUnmatchedDenyEngineAndUnknownAllowEngine() doReturn(false).when(spyEngine).matches(eq(condition5), any(Activation.class)); doReturn(false).when(spyEngine).matches(eq(condition6), any(Activation.class)); evaluateResult = spyEngine.evaluate(args); - assertEquals(evaluateResult.getDecision(), AuthorizationDecision.Output.UNKNOWN); - assertEquals(evaluateResult.getPolicyNames().size(), 2); - assertTrue(evaluateResult.getPolicyNames().contains("Policy 2")); - assertTrue(evaluateResult.getPolicyNames().contains("Policy 3")); + assertThat(evaluateResult.getDecision()).isEqualTo(AuthorizationDecision.Output.UNKNOWN); + assertThat(evaluateResult.getPolicyNames()).hasSize(2); + assertThat(evaluateResult.getPolicyNames()).contains("Policy 2"); + assertThat(evaluateResult.getPolicyNames()).contains("Policy 3"); } /** @@ -469,7 +463,7 @@ public void testUnmatchedEnginePair() throws InterpreterException { doReturn(false).when(spyEngine).matches(eq(condition5), any(Activation.class)); doReturn(false).when(spyEngine).matches(eq(condition6), any(Activation.class)); evaluateResult = spyEngine.evaluate(args); - assertEquals(evaluateResult.getDecision(), AuthorizationDecision.Output.DENY); - assertEquals(evaluateResult.getPolicyNames().size(), 0); + assertThat(evaluateResult.getDecision()).isEqualTo(AuthorizationDecision.Output.DENY); + assertThat(evaluateResult.getPolicyNames()).isEmpty(); } } diff --git a/xds/src/test/java/io/grpc/xds/internal/rbac/engine/AuthzEngineTest.java b/xds/src/test/java/io/grpc/xds/internal/rbac/engine/AuthzEngineTest.java index 17407d3ce18..9716260af28 100644 --- a/xds/src/test/java/io/grpc/xds/internal/rbac/engine/AuthzEngineTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/rbac/engine/AuthzEngineTest.java @@ -68,7 +68,6 @@ public class AuthzEngineTest { private AuthorizationEngine engine; private RBAC rbacDeny; private RBAC rbacAllow; - private Expr expr; private Object result; @Before @@ -130,7 +129,7 @@ public void failToCreateEngineIfRbacPairOfDenyDeny() { public void testCelInterface() throws InterpreterException { engine = new AuthorizationEngine(rbacAllow); when(interpretable.eval(any(Activation.class))).thenReturn(true); - expr = Expr.newBuilder().build(); + Expr expr = Expr.getDefaultInstance(); result = engine.matches(expr, activation); assertThat(messageProvider).isNotNull(); assertThat(dispatcher).isNotNull(); diff --git a/xds/src/test/java/io/grpc/xds/internal/rbac/engine/EvaluateArgsTest.java b/xds/src/test/java/io/grpc/xds/internal/rbac/engine/EvaluateArgsTest.java index 3fa09bda451..8f908f91888 100644 --- a/xds/src/test/java/io/grpc/xds/internal/rbac/engine/EvaluateArgsTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/rbac/engine/EvaluateArgsTest.java @@ -106,7 +106,7 @@ public void testGenerateEnvoyAttributes() { verify(spyArgs, times(1)).getConnectionUriSanPeerCertificate(); verify(spyArgs, times(1)).getSourcePrincipal(); } - + @Test public void testEvaluateArgsAccessorFunctions() { // Set up args and call. diff --git a/xds/src/test/java/io/grpc/xds/internal/sds/CommonTlsContextTestsUtil.java b/xds/src/test/java/io/grpc/xds/internal/sds/CommonTlsContextTestsUtil.java index 8c150d0ccd0..dad594f2783 100644 --- a/xds/src/test/java/io/grpc/xds/internal/sds/CommonTlsContextTestsUtil.java +++ b/xds/src/test/java/io/grpc/xds/internal/sds/CommonTlsContextTestsUtil.java @@ -33,6 +33,7 @@ import io.envoyproxy.envoy.config.core.v3.GrpcService; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CertificateValidationContext; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext; +import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext.CertificateProviderInstance; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext.CombinedCertificateValidationContext; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.SdsSecretConfig; @@ -497,19 +498,20 @@ private static CommonTlsContext.Builder addCertificateValidationContext( String rootCertName, CertificateValidationContext staticCertValidationContext) { if (rootInstanceName != null) { - CommonTlsContext.CertificateProviderInstance.Builder providerInstanceBuilder = - CommonTlsContext.CertificateProviderInstance.newBuilder() + CertificateProviderInstance providerInstance = + CertificateProviderInstance.newBuilder() .setInstanceName(rootInstanceName) - .setCertificateName(rootCertName); + .setCertificateName(rootCertName) + .build(); if (staticCertValidationContext != null) { CombinedCertificateValidationContext combined = CombinedCertificateValidationContext.newBuilder() .setDefaultValidationContext(staticCertValidationContext) - .setValidationContextCertificateProviderInstance(providerInstanceBuilder) + .setValidationContextCertificateProviderInstance(providerInstance) .build(); return builder.setCombinedValidationContext(combined); } - builder = builder.setValidationContextCertificateProviderInstance(providerInstanceBuilder); + builder = builder.setValidationContextCertificateProviderInstance(providerInstance); } return builder; } From 1260db3305074e9537513be48e275bd880e0277a Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Fri, 28 Aug 2020 18:08:15 -0700 Subject: [PATCH 07/86] xds: eliminate cluster name change logic in CDS LB policy and reimplement tests (#7356) Eliminated the logic in CDS LB policy that handles CDS config change (aka, cluster name change). A CDS LB policy should be used for a single cluster, the cluster should be effectively final. If the upstream LB policy needs to change routing cluster, it should create separate CDS LB policies, one for each cluster. This change also reimplemented the unit tests for CDS LB policy. --- .../java/io/grpc/xds/CdsLoadBalancer.java | 193 ++-- .../java/io/grpc/xds/CdsLoadBalancerTest.java | 888 ++++++++---------- 2 files changed, 468 insertions(+), 613 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java index d0b9248ac42..cf9e7991df6 100644 --- a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java @@ -32,7 +32,6 @@ import io.grpc.internal.ObjectPool; import io.grpc.internal.ServiceConfigUtil.PolicySelection; import io.grpc.util.ForwardingLoadBalancerHelper; -import io.grpc.util.GracefulSwitchLoadBalancer; import io.grpc.xds.CdsLoadBalancerProvider.CdsConfig; import io.grpc.xds.EdsLoadBalancerProvider.EdsConfig; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; @@ -45,31 +44,24 @@ import io.grpc.xds.internal.sds.TlsContextManagerImpl; import java.util.ArrayList; import java.util.List; -import java.util.Objects; -import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nullable; /** - * Load balancer for cds_experimental LB policy. + * Load balancer for cds_experimental LB policy. One instance per cluster. */ -public final class CdsLoadBalancer extends LoadBalancer { +final class CdsLoadBalancer extends LoadBalancer { private final XdsLogger logger; + private final Helper helper; private final LoadBalancerRegistry lbRegistry; - private final GracefulSwitchLoadBalancer switchingLoadBalancer; private final TlsContextManager tlsContextManager; // TODO(sanjaypujare): remove once xds security is released private boolean enableXdsSecurity; private static final String XDS_SECURITY_ENV_VAR = "GRPC_XDS_EXPERIMENTAL_SECURITY_SUPPORT"; - - // The following fields become non-null once handleResolvedAddresses() successfully. - - // Most recent cluster name. - @Nullable private String clusterName; - @Nullable private ObjectPool xdsClientPool; - @Nullable private XdsClient xdsClient; + private ChildLbState childLbState; + private ResolvedAddresses resolvedAddresses; CdsLoadBalancer(Helper helper) { this(helper, LoadBalancerRegistry.getDefaultRegistry(), TlsContextManagerImpl.getInstance()); @@ -78,9 +70,8 @@ public final class CdsLoadBalancer extends LoadBalancer { @VisibleForTesting CdsLoadBalancer(Helper helper, LoadBalancerRegistry lbRegistry, TlsContextManager tlsContextManager) { - checkNotNull(helper, "helper"); + this.helper = checkNotNull(helper, "helper"); this.lbRegistry = lbRegistry; - this.switchingLoadBalancer = new GracefulSwitchLoadBalancer(helper); this.tlsContextManager = tlsContextManager; logger = XdsLogger.withLogId(InternalLogId.allocate("cds-lb", helper.getAuthority())); logger.log(XdsLogLevel.INFO, "Created"); @@ -88,33 +79,32 @@ public final class CdsLoadBalancer extends LoadBalancer { @Override public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { - logger.log(XdsLogLevel.DEBUG, "Received resolution result: {0}", resolvedAddresses); - if (xdsClientPool == null) { - xdsClientPool = resolvedAddresses.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL); - checkNotNull(xdsClientPool, "missing xDS client pool"); - xdsClient = xdsClientPool.getObject(); + if (clusterName != null) { + return; } - + logger.log(XdsLogLevel.DEBUG, "Received resolution result: {0}", resolvedAddresses); + this.resolvedAddresses = resolvedAddresses; + xdsClientPool = resolvedAddresses.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL); + checkNotNull(xdsClientPool, "missing xDS client pool"); + xdsClient = xdsClientPool.getObject(); Object lbConfig = resolvedAddresses.getLoadBalancingPolicyConfig(); checkNotNull(lbConfig, "missing CDS lb config"); CdsConfig newCdsConfig = (CdsConfig) lbConfig; logger.log( XdsLogLevel.INFO, "Received CDS lb config: cluster={0}", newCdsConfig.name); - - // If cluster is changed, do a graceful switch. - if (!newCdsConfig.name.equals(clusterName)) { - LoadBalancer.Factory clusterBalancerFactory = new ClusterBalancerFactory(newCdsConfig.name); - switchingLoadBalancer.switchTo(clusterBalancerFactory); - } - switchingLoadBalancer.handleResolvedAddresses(resolvedAddresses); clusterName = newCdsConfig.name; + childLbState = new ChildLbState(); } @Override public void handleNameResolutionError(Status error) { logger.log(XdsLogLevel.WARNING, "Received name resolution error: {0}", error); - switchingLoadBalancer.handleNameResolutionError(error); + if (childLbState != null) { + childLbState.propagateError(error); + } else { + helper.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(error)); + } } @Override @@ -125,7 +115,9 @@ public boolean canHandleEmptyAddressListFromNameResolution() { @Override public void shutdown() { logger.log(XdsLogLevel.INFO, "Shutdown"); - switchingLoadBalancer.shutdown(); + if (childLbState != null) { + childLbState.shutdown(); + } if (xdsClientPool != null) { xdsClientPool.returnObject(xdsClient); } @@ -142,107 +134,25 @@ void setXdsSecurity(boolean enable) { enableXdsSecurity = enable; } - /** - * A load balancer factory that provides a load balancer for a given cluster. - */ - private final class ClusterBalancerFactory extends LoadBalancer.Factory { - - final String clusterName; - - ClusterBalancerFactory(String clusterName) { - this.clusterName = clusterName; - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof ClusterBalancerFactory)) { - return false; - } - ClusterBalancerFactory that = (ClusterBalancerFactory) o; - return clusterName.equals(that.clusterName); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), clusterName); - } - - @Override - public LoadBalancer newLoadBalancer(final Helper helper) { - return new LoadBalancer() { - // Becomes non-null once handleResolvedAddresses() successfully. - // Assigned at most once. - @Nullable - ClusterWatcherImpl clusterWatcher; - - @Override - public void handleNameResolutionError(Status error) { - if (clusterWatcher == null || clusterWatcher.edsBalancer == null) { - // Go into TRANSIENT_FAILURE if we have not yet received any cluster resource. - // Otherwise, we keep running with the data we had previously. - helper.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(error)); - } - } - - @Override - public boolean canHandleEmptyAddressListFromNameResolution() { - return true; - } - - @Override - public void shutdown() { - if (clusterWatcher != null) { - if (clusterWatcher.edsBalancer != null) { - clusterWatcher.edsBalancer.shutdown(); - } - xdsClient.cancelClusterDataWatch(clusterName, clusterWatcher); - logger.log( - XdsLogLevel.INFO, - "Cancelled cluster watcher on {0} with xDS client {1}", - clusterName, xdsClient); - } - } - - @Override - public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { - if (clusterWatcher == null) { - clusterWatcher = new ClusterWatcherImpl(helper, resolvedAddresses); - logger.log( - XdsLogLevel.INFO, - "Start cluster watcher on {0} with xDS client {1}", - clusterName, xdsClient); - xdsClient.watchClusterData(clusterName, clusterWatcher); - } - } - }; - } - } - - private static final class EdsLoadBalancingHelper extends ForwardingLoadBalancerHelper { - private final Helper delegate; - private final AtomicReference sslContextProvider; - - EdsLoadBalancingHelper(Helper helper, - AtomicReference sslContextProvider) { - this.delegate = helper; - this.sslContextProvider = sslContextProvider; - } + private final class ChannelSecurityLbHelper extends ForwardingLoadBalancerHelper { + @Nullable + private SslContextProvider sslContextProvider; @Override public Subchannel createSubchannel(CreateSubchannelArgs createSubchannelArgs) { - if (sslContextProvider.get() != null) { + if (sslContextProvider != null) { createSubchannelArgs = createSubchannelArgs .toBuilder() .setAddresses( addUpstreamTlsContext(createSubchannelArgs.getAddresses(), - sslContextProvider.get().getUpstreamTlsContext())) + sslContextProvider.getUpstreamTlsContext())) .build(); } - return delegate.createSubchannel(createSubchannelArgs); + return delegate().createSubchannel(createSubchannelArgs); } - private static List addUpstreamTlsContext( + private List addUpstreamTlsContext( List addresses, UpstreamTlsContext upstreamTlsContext) { if (upstreamTlsContext == null || addresses == null) { @@ -264,22 +174,19 @@ private static List addUpstreamTlsContext( @Override protected Helper delegate() { - return delegate; + return helper; } } - private final class ClusterWatcherImpl implements ClusterWatcher { - - final EdsLoadBalancingHelper helper; - final ResolvedAddresses resolvedAddresses; - + private final class ChildLbState implements ClusterWatcher { + private final ChannelSecurityLbHelper lbHelper = new ChannelSecurityLbHelper(); @Nullable LoadBalancer edsBalancer; - ClusterWatcherImpl(Helper helper, ResolvedAddresses resolvedAddresses) { - this.helper = new EdsLoadBalancingHelper(helper, - new AtomicReference()); - this.resolvedAddresses = resolvedAddresses; + private ChildLbState() { + xdsClient.watchClusterData(clusterName, this); + logger.log(XdsLogLevel.INFO, + "Started watcher for cluster {0} with xDS client {1}", clusterName, xdsClient); } @Override @@ -308,7 +215,7 @@ public void onClusterChanged(ClusterUpdate newUpdate) { updateSslContextProvider(newUpdate.getUpstreamTlsContext()); } if (edsBalancer == null) { - edsBalancer = lbRegistry.getProvider(EDS_POLICY_NAME).newLoadBalancer(helper); + edsBalancer = lbRegistry.getProvider(EDS_POLICY_NAME).newLoadBalancer(lbHelper); } edsBalancer.handleResolvedAddresses( resolvedAddresses.toBuilder().setLoadBalancingPolicyConfig(edsConfig).build()); @@ -316,8 +223,7 @@ public void onClusterChanged(ClusterUpdate newUpdate) { /** For new UpstreamTlsContext value, release old SslContextProvider. */ private void updateSslContextProvider(UpstreamTlsContext newUpstreamTlsContext) { - SslContextProvider oldSslContextProvider = - helper.sslContextProvider.get(); + SslContextProvider oldSslContextProvider = lbHelper.sslContextProvider; if (oldSslContextProvider != null) { UpstreamTlsContext oldUpstreamTlsContext = oldSslContextProvider.getUpstreamTlsContext(); @@ -327,11 +233,10 @@ private void updateSslContextProvider(UpstreamTlsContext newUpstreamTlsContext) tlsContextManager.releaseClientSslContextProvider(oldSslContextProvider); } if (newUpstreamTlsContext != null) { - SslContextProvider newSslContextProvider = + lbHelper.sslContextProvider = tlsContextManager.findOrCreateClientSslContextProvider(newUpstreamTlsContext); - helper.sslContextProvider.set(newSslContextProvider); } else { - helper.sslContextProvider.set(null); + lbHelper.sslContextProvider = null; } } @@ -360,6 +265,22 @@ public void onError(Status error) { helper.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(error)); } } - } + void shutdown() { + xdsClient.cancelClusterDataWatch(clusterName, this); + logger.log(XdsLogLevel.INFO, + "Cancelled watcher for cluster {0} with xDS client {1}", clusterName, xdsClient); + if (edsBalancer != null) { + edsBalancer.shutdown(); + } + } + + void propagateError(Status error) { + if (edsBalancer != null) { + edsBalancer.handleNameResolutionError(error); + } else { + helper.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(error)); + } + } + } } diff --git a/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java index 9a9bb69241f..1f2261cbdef 100644 --- a/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java @@ -17,26 +17,10 @@ package io.grpc.xds; import static com.google.common.truth.Truth.assertThat; -import static io.grpc.ConnectivityState.CONNECTING; -import static io.grpc.ConnectivityState.READY; -import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; -import static io.grpc.xds.XdsLbPolicies.EDS_POLICY_NAME; -import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.BAD_CLIENT_KEY_FILE; -import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.BAD_CLIENT_PEM_FILE; -import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.CA_PEM_FILE; -import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.CLIENT_KEY_FILE; -import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.CLIENT_PEM_FILE; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.same; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; -import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; import io.grpc.Attributes; import io.grpc.ConnectivityState; import io.grpc.EquivalentAddressGroup; @@ -46,38 +30,34 @@ import io.grpc.LoadBalancer.PickResult; import io.grpc.LoadBalancer.PickSubchannelArgs; import io.grpc.LoadBalancer.ResolvedAddresses; +import io.grpc.LoadBalancer.Subchannel; import io.grpc.LoadBalancer.SubchannelPicker; import io.grpc.LoadBalancerProvider; import io.grpc.LoadBalancerRegistry; -import io.grpc.NameResolver.ConfigOrError; +import io.grpc.ManagedChannel; +import io.grpc.NameResolver; import io.grpc.Status; import io.grpc.Status.Code; import io.grpc.SynchronizationContext; -import io.grpc.internal.FakeClock; -import io.grpc.internal.ServiceConfigUtil.PolicySelection; +import io.grpc.internal.ObjectPool; import io.grpc.xds.CdsLoadBalancerProvider.CdsConfig; import io.grpc.xds.EdsLoadBalancerProvider.EdsConfig; +import io.grpc.xds.EnvoyServerProtoData.DownstreamTlsContext; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; -import io.grpc.xds.XdsClient.ClusterUpdate; -import io.grpc.xds.XdsClient.ClusterWatcher; -import io.grpc.xds.XdsClient.RefCountedXdsClientObjectPool; -import io.grpc.xds.XdsClient.XdsClientFactory; import io.grpc.xds.internal.sds.CommonTlsContextTestsUtil; import io.grpc.xds.internal.sds.SslContextProvider; import io.grpc.xds.internal.sds.TlsContextManager; -import java.net.InetSocketAddress; -import java.util.ArrayDeque; +import java.net.SocketAddress; import java.util.ArrayList; -import java.util.Deque; -import java.util.HashMap; +import java.util.Collections; import java.util.List; -import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -import org.mockito.ArgumentCaptor; -import org.mockito.Mock; import org.mockito.MockitoAnnotations; /** @@ -85,44 +65,323 @@ */ @RunWith(JUnit4.class) public class CdsLoadBalancerTest { - - private final RefCountedXdsClientObjectPool xdsClientPool = new RefCountedXdsClientObjectPool( - new XdsClientFactory() { + private static final String AUTHORITY = "api.google.com"; + private static final String CLUSTER = "cluster-foo.googleapis.com"; + private final SynchronizationContext syncContext = new SynchronizationContext( + new Thread.UncaughtExceptionHandler() { @Override - XdsClient createXdsClient() { - xdsClient = mock(XdsClient.class); - return xdsClient; + public void uncaughtException(Thread t, Throwable e) { + throw new AssertionError(e); } + }); + private final List childBalancers = new ArrayList<>(); + private final FakeXdsClient xdsClient = new FakeXdsClient(); + private final TlsContextManager tlsContextManager = new FakeTlsContextManager(); + private LoadBalancer.Helper helper = new FakeLbHelper(); + private int xdsClientRefs; + private ConnectivityState currentState; + private SubchannelPicker currentPicker; + private CdsLoadBalancer loadBalancer; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + + LoadBalancerRegistry registry = new LoadBalancerRegistry(); + registry.register(new FakeLoadBalancerProvider(XdsLbPolicies.EDS_POLICY_NAME)); + registry.register(new FakeLoadBalancerProvider("round_robin")); + ObjectPool xdsClientPool = new ObjectPool() { + @Override + public XdsClient getObject() { + xdsClientRefs++; + return xdsClient; } - ); - private final LoadBalancerRegistry lbRegistry = new LoadBalancerRegistry(); - private final LoadBalancerProvider fakeEdsLoadBlancerProvider = new LoadBalancerProvider() { + @Override + public XdsClient returnObject(Object object) { + assertThat(xdsClientRefs).isGreaterThan(0); + xdsClientRefs--; + return null; + } + }; + loadBalancer = new CdsLoadBalancer(helper, registry, tlsContextManager); + loadBalancer.handleResolvedAddresses( + ResolvedAddresses.newBuilder() + .setAddresses(Collections.emptyList()) + .setLoadBalancingPolicyConfig(new CdsConfig(CLUSTER)) + .setAttributes( + Attributes.newBuilder().set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool).build()) + .build()); + assertThat(xdsClient.watcher).isNotNull(); + } + + @After + public void tearDown() { + loadBalancer.shutdown(); + assertThat(xdsClient.watcher).isNull(); + assertThat(xdsClientRefs).isEqualTo(0); + } + + + @Test + public void receiveFirstClusterResourceInfo() { + xdsClient.deliverClusterInfo(null, null); + assertThat(childBalancers).hasSize(1); + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); + assertThat(childBalancer.name).isEqualTo(XdsLbPolicies.EDS_POLICY_NAME); + assertThat(childBalancer.config).isNotNull(); + EdsConfig edsConfig = (EdsConfig) childBalancer.config; + assertThat(edsConfig.clusterName).isEqualTo(CLUSTER); + assertThat(edsConfig.edsServiceName).isNull(); + assertThat(edsConfig.lrsServerName).isNull(); + assertThat(edsConfig.endpointPickingPolicy.getProvider().getPolicyName()) + .isEqualTo("round_robin"); + } + + @Test + public void clusterResourceNeverExist() { + xdsClient.deliverResourceNotFound(); + assertThat(childBalancers).isEmpty(); + assertThat(currentState).isEqualTo(ConnectivityState.TRANSIENT_FAILURE); + PickResult result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); + assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAVAILABLE); + assertThat(result.getStatus().getDescription()) + .isEqualTo("Resource " + CLUSTER + " is unavailable"); + } + + @Test + public void clusterResourceRemoved() { + xdsClient.deliverClusterInfo(null, null); + assertThat(childBalancers).hasSize(1); + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); + assertThat(childBalancer.shutdown).isFalse(); + + xdsClient.deliverResourceNotFound(); + assertThat(childBalancer.shutdown).isTrue(); + assertThat(currentState).isEqualTo(ConnectivityState.TRANSIENT_FAILURE); + PickResult result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); + assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAVAILABLE); + assertThat(result.getStatus().getDescription()) + .isEqualTo("Resource " + CLUSTER + " is unavailable"); + } + + @Test + public void clusterResourceUpdated() { + xdsClient.deliverClusterInfo(null, null); + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); + EdsConfig edsConfig = (EdsConfig) childBalancer.config; + assertThat(edsConfig.clusterName).isEqualTo(CLUSTER); + assertThat(edsConfig.edsServiceName).isNull(); + assertThat(edsConfig.lrsServerName).isNull(); + assertThat(edsConfig.endpointPickingPolicy.getProvider().getPolicyName()) + .isEqualTo("round_robin"); + + String edsService = "service-bar.googleapis.com"; + String loadReportServer = "lrs-server.googleapis.com"; + xdsClient.deliverClusterInfo(edsService, loadReportServer); + assertThat(childBalancers).containsExactly(childBalancer); + edsConfig = (EdsConfig) childBalancer.config; + assertThat(edsConfig.clusterName).isEqualTo(CLUSTER); + assertThat(edsConfig.edsServiceName).isEqualTo(edsService); + assertThat(edsConfig.lrsServerName).isEqualTo(loadReportServer); + assertThat(edsConfig.endpointPickingPolicy.getProvider().getPolicyName()) + .isEqualTo("round_robin"); + } + + @Test + public void receiveClusterResourceInfoWithUpstreamTlsContext() { + loadBalancer.setXdsSecurity(true); + UpstreamTlsContext upstreamTlsContext = + CommonTlsContextTestsUtil.buildUpstreamTlsContextFromFilenames( + CommonTlsContextTestsUtil.CLIENT_KEY_FILE, + CommonTlsContextTestsUtil.CLIENT_PEM_FILE, + CommonTlsContextTestsUtil.CA_PEM_FILE); + xdsClient.deliverClusterInfo(null, null, upstreamTlsContext); + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); + List addresses = createEndpointAddresses(2); + CreateSubchannelArgs args = + CreateSubchannelArgs.newBuilder() + .setAddresses(addresses) + .build(); + Subchannel subchannel = childBalancer.helper.createSubchannel(args); + for (EquivalentAddressGroup eag : subchannel.getAllAddresses()) { + assertThat(eag.getAttributes().get(XdsAttributes.ATTR_UPSTREAM_TLS_CONTEXT)) + .isEqualTo(upstreamTlsContext); + } + + xdsClient.deliverClusterInfo(null, null); + subchannel = childBalancer.helper.createSubchannel(args); + for (EquivalentAddressGroup eag : subchannel.getAllAddresses()) { + assertThat(eag.getAttributes().get(XdsAttributes.ATTR_UPSTREAM_TLS_CONTEXT)).isNull(); + } + + upstreamTlsContext = + CommonTlsContextTestsUtil.buildUpstreamTlsContextFromFilenames( + CommonTlsContextTestsUtil.BAD_CLIENT_KEY_FILE, + CommonTlsContextTestsUtil.BAD_CLIENT_PEM_FILE, + CommonTlsContextTestsUtil.CA_PEM_FILE); + xdsClient.deliverClusterInfo(null, null, upstreamTlsContext); + subchannel = childBalancer.helper.createSubchannel(args); + for (EquivalentAddressGroup eag : subchannel.getAllAddresses()) { + assertThat(eag.getAttributes().get(XdsAttributes.ATTR_UPSTREAM_TLS_CONTEXT)) + .isEqualTo(upstreamTlsContext); + } + } + + @Test + public void subchannelStatePropagateFromDownstreamToUpstream() { + xdsClient.deliverClusterInfo(null, null); + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); + List addresses = createEndpointAddresses(2); + CreateSubchannelArgs args = + CreateSubchannelArgs.newBuilder() + .setAddresses(addresses) + .build(); + Subchannel subchannel = childBalancer.helper.createSubchannel(args); + childBalancer.deliverSubchannelState(subchannel, ConnectivityState.READY); + assertThat(currentState).isEqualTo(ConnectivityState.READY); + assertThat(currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)).getSubchannel()) + .isSameInstanceAs(subchannel); + } + + @Test + public void clusterDiscoveryError_beforeChildPolicyInstantiated_propagateToUpstream() { + xdsClient.deliverError(Status.UNAUTHENTICATED.withDescription("permission denied")); + assertThat(currentState).isEqualTo(ConnectivityState.TRANSIENT_FAILURE); + PickResult result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); + assertThat(result.getStatus().isOk()).isFalse(); + assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAUTHENTICATED); + assertThat(result.getStatus().getDescription()).isEqualTo("permission denied"); + } + + @Test + public void clusterDiscoveryError_afterChildPolicyInstantiated_keepUsingCurrentCluster() { + xdsClient.deliverClusterInfo(null, null); + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); + xdsClient.deliverError(Status.UNAVAILABLE.withDescription("unreachable")); + assertThat(currentState).isNull(); + assertThat(currentPicker).isNull(); + assertThat(childBalancer.shutdown).isFalse(); + } + + @Test + public void nameResolutionError_beforeChildPolicyInstantiated_returnErrorPickerToUpstream() { + loadBalancer.handleNameResolutionError( + Status.UNIMPLEMENTED.withDescription("not found")); + assertThat(currentState).isEqualTo(ConnectivityState.TRANSIENT_FAILURE); + PickResult result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); + assertThat(result.getStatus().isOk()).isFalse(); + assertThat(result.getStatus().getCode()).isEqualTo(Code.UNIMPLEMENTED); + assertThat(result.getStatus().getDescription()).isEqualTo("not found"); + } + + @Test + public void nameResolutionError_afterChildPolicyInstantiated_propagateToDownstream() { + xdsClient.deliverClusterInfo(null, null); + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); + loadBalancer.handleNameResolutionError( + Status.UNAVAILABLE.withDescription("cannot reach server")); + assertThat(childBalancer.upstreamError.getCode()).isEqualTo(Code.UNAVAILABLE); + assertThat(childBalancer.upstreamError.getDescription()) + .isEqualTo("cannot reach server"); + } + + private static List createEndpointAddresses(int n) { + List list = new ArrayList<>(); + for (int i = 0; i < n; i++) { + list.add(new EquivalentAddressGroup(mock(SocketAddress.class))); + } + return list; + } + + private final class FakeXdsClient extends XdsClient { + private ClusterWatcher watcher; + @Override - public boolean isAvailable() { - return true; + void watchClusterData(String clusterName, ClusterWatcher watcher) { + assertThat(clusterName).isEqualTo(CLUSTER); + this.watcher = watcher; } @Override - public int getPriority() { - return 5; + void cancelClusterDataWatch(String clusterName, ClusterWatcher watcher) { + assertThat(clusterName).isEqualTo(CLUSTER); + assertThat(watcher).isSameInstanceAs(this.watcher); + this.watcher = null; } @Override - public String getPolicyName() { - return EDS_POLICY_NAME; + void shutdown() { + // no-op + } + + void deliverClusterInfo( + @Nullable final String edsServiceName, @Nullable final String lrsServerName) { + syncContext.execute(new Runnable() { + @Override + public void run() { + watcher.onClusterChanged( + ClusterUpdate.newBuilder() + .setClusterName(CLUSTER) + .setEdsServiceName(edsServiceName) + .setLbPolicy("round_robin") // only supported policy + .setLrsServerName(lrsServerName) + .build()); + } + }); + } + + void deliverClusterInfo( + @Nullable final String edsServiceName, @Nullable final String lrsServerName, + final UpstreamTlsContext tlsContext) { + syncContext.execute(new Runnable() { + @Override + public void run() { + watcher.onClusterChanged( + ClusterUpdate.newBuilder() + .setClusterName(CLUSTER) + .setEdsServiceName(edsServiceName) + .setLbPolicy("round_robin") // only supported policy + .setLrsServerName(lrsServerName) + .setUpstreamTlsContext(tlsContext) + .build()); + } + }); + } + + void deliverResourceNotFound() { + syncContext.execute(new Runnable() { + @Override + public void run() { + watcher.onResourceDoesNotExist(CLUSTER); + } + }); + } + + void deliverError(final Status error) { + syncContext.execute(new Runnable() { + @Override + public void run() { + watcher.onError(error); + } + }); + } + } + + private final class FakeLoadBalancerProvider extends LoadBalancerProvider { + private final String policyName; + + FakeLoadBalancerProvider(String policyName) { + this.policyName = policyName; } @Override public LoadBalancer newLoadBalancer(Helper helper) { - edsLbHelpers.add(helper); - LoadBalancer edsLoadBalancer = mock(LoadBalancer.class); - edsLoadBalancers.add(edsLoadBalancer); - return edsLoadBalancer; + FakeLoadBalancer balancer = new FakeLoadBalancer(policyName, helper); + childBalancers.add(balancer); + return balancer; } - }; - private final LoadBalancerProvider fakeRoundRobinLbProvider = new LoadBalancerProvider() { @Override public boolean isAvailable() { return true; @@ -130,463 +389,138 @@ public boolean isAvailable() { @Override public int getPriority() { - return 5; + return 0; // doesn't matter } @Override public String getPolicyName() { - return "round_robin"; + return policyName; + } + } + + private final class FakeLoadBalancer extends LoadBalancer { + private final String name; + private final Helper helper; + private Object config; + private Status upstreamError; + private boolean shutdown; + + FakeLoadBalancer(String name, Helper helper) { + this.name = name; + this.helper = helper; } @Override - public LoadBalancer newLoadBalancer(Helper helper) { - return mock(LoadBalancer.class); + public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { + config = resolvedAddresses.getLoadBalancingPolicyConfig(); } @Override - public ConfigOrError parseLoadBalancingPolicyConfig( - Map rawLoadBalancingPolicyConfig) { - return ConfigOrError.fromConfig("fake round robin config"); + public void handleNameResolutionError(Status error) { + upstreamError = error; } - }; - private final SynchronizationContext syncContext = new SynchronizationContext( - new Thread.UncaughtExceptionHandler() { + @Override + public void shutdown() { + shutdown = true; + childBalancers.remove(this); + } + + void deliverSubchannelState(final Subchannel subchannel, ConnectivityState state) { + SubchannelPicker picker = new SubchannelPicker() { @Override - public void uncaughtException(Thread t, Throwable e) { - throw new AssertionError(e); + public PickResult pickSubchannel(PickSubchannelArgs args) { + return PickResult.withSubchannel(subchannel); } - }); - - private final FakeClock fakeClock = new FakeClock(); - private final Deque edsLoadBalancers = new ArrayDeque<>(); - private final Deque edsLbHelpers = new ArrayDeque<>(); - - @Mock - private Helper helper; - - private LoadBalancer cdsLoadBalancer; - private XdsClient xdsClient; - - @Mock - private TlsContextManager mockTlsContextManager; - - @Before - public void setUp() { - MockitoAnnotations.initMocks(this); - - doReturn(syncContext).when(helper).getSynchronizationContext(); - doReturn(fakeClock.getScheduledExecutorService()).when(helper).getScheduledExecutorService(); - lbRegistry.register(fakeEdsLoadBlancerProvider); - lbRegistry.register(fakeRoundRobinLbProvider); - cdsLoadBalancer = new CdsLoadBalancer(helper, lbRegistry, mockTlsContextManager); + }; + helper.updateBalancingState(state, picker); + } } - @Test - public void canHandleEmptyAddressListFromNameResolution() { - assertThat(cdsLoadBalancer.canHandleEmptyAddressListFromNameResolution()).isTrue(); - } + private final class FakeLbHelper extends LoadBalancer.Helper { - @Test - public void handleResolutionErrorBeforeOrAfterCdsWorking() { - ResolvedAddresses resolvedAddresses1 = ResolvedAddresses.newBuilder() - .setAddresses(ImmutableList.of()) - .setAttributes(Attributes.newBuilder() - .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) - .build()) - .setLoadBalancingPolicyConfig(new CdsConfig("foo.googleapis.com")) - .build(); - cdsLoadBalancer.handleResolvedAddresses(resolvedAddresses1); - ArgumentCaptor clusterWatcherCaptor1 = ArgumentCaptor.forClass(null); - verify(xdsClient).watchClusterData(eq("foo.googleapis.com"), clusterWatcherCaptor1.capture()); - ClusterWatcher clusterWatcher1 = clusterWatcherCaptor1.getValue(); - - // handleResolutionError() before receiving any CDS response. - cdsLoadBalancer.handleNameResolutionError(Status.DATA_LOSS.withDescription("fake status")); - verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); - - // CDS response received. - clusterWatcher1.onClusterChanged( - ClusterUpdate.newBuilder() - .setClusterName("foo.googleapis.com") - .setEdsServiceName("edsServiceFoo.googleapis.com") - .setLbPolicy("round_robin") - .build()); - verify(helper).updateBalancingState(eq(CONNECTING), any(SubchannelPicker.class)); + @Override + public void updateBalancingState( + @Nonnull ConnectivityState newState, @Nonnull SubchannelPicker newPicker) { + currentState = newState; + currentPicker = newPicker; + } - // handleResolutionError() after receiving CDS response. - cdsLoadBalancer.handleNameResolutionError(Status.DATA_LOSS.withDescription("fake status")); - // No more TRANSIENT_FAILURE. - verify(helper, times(1)).updateBalancingState( - eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); - } + @Override + public Subchannel createSubchannel(CreateSubchannelArgs args) { + return new FakeSubchannel(args.getAddresses()); + } - @Test - public void handleCdsConfigUpdate() { - assertThat(xdsClient).isNull(); - ResolvedAddresses resolvedAddresses1 = ResolvedAddresses.newBuilder() - .setAddresses(ImmutableList.of()) - .setAttributes(Attributes.newBuilder() - .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) - .build()) - .setLoadBalancingPolicyConfig(new CdsConfig("foo.googleapis.com")) - .build(); - cdsLoadBalancer.handleResolvedAddresses(resolvedAddresses1); - - ArgumentCaptor clusterWatcherCaptor1 = ArgumentCaptor.forClass(null); - verify(xdsClient).watchClusterData(eq("foo.googleapis.com"), clusterWatcherCaptor1.capture()); - - ClusterWatcher clusterWatcher1 = clusterWatcherCaptor1.getValue(); - clusterWatcher1.onClusterChanged( - ClusterUpdate.newBuilder() - .setClusterName("foo.googleapis.com") - .setEdsServiceName("edsServiceFoo.googleapis.com") - .setLbPolicy("round_robin") - .build()); + @Override + public ManagedChannel createOobChannel(EquivalentAddressGroup eag, String authority) { + throw new UnsupportedOperationException("should not be called"); + } - assertThat(edsLbHelpers).hasSize(1); - assertThat(edsLoadBalancers).hasSize(1); - Helper edsLbHelper1 = edsLbHelpers.poll(); - LoadBalancer edsLoadBalancer1 = edsLoadBalancers.poll(); - ArgumentCaptor resolvedAddressesCaptor1 = ArgumentCaptor.forClass(null); - verify(edsLoadBalancer1).handleResolvedAddresses(resolvedAddressesCaptor1.capture()); - PolicySelection roundRobinPolicy = new PolicySelection( - fakeRoundRobinLbProvider, new HashMap(), "fake round robin config"); - EdsConfig expectedEdsConfig = new EdsConfig( - "foo.googleapis.com", - "edsServiceFoo.googleapis.com", - null, - roundRobinPolicy); - ResolvedAddresses resolvedAddressesFoo = resolvedAddressesCaptor1.getValue(); - assertThat(resolvedAddressesFoo.getLoadBalancingPolicyConfig()).isEqualTo(expectedEdsConfig); - assertThat(resolvedAddressesFoo.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL)) - .isSameInstanceAs(xdsClientPool); - - SubchannelPicker picker1 = mock(SubchannelPicker.class); - edsLbHelper1.updateBalancingState(ConnectivityState.READY, picker1); - verify(helper).updateBalancingState(ConnectivityState.READY, picker1); - - ResolvedAddresses resolvedAddresses2 = ResolvedAddresses.newBuilder() - .setAddresses(ImmutableList.of()) - .setAttributes(Attributes.newBuilder() - .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) - .build()) - .setLoadBalancingPolicyConfig(new CdsConfig("bar.googleapis.com")) - .build(); - cdsLoadBalancer.handleResolvedAddresses(resolvedAddresses2); - - ArgumentCaptor clusterWatcherCaptor2 = ArgumentCaptor.forClass(null); - verify(xdsClient).watchClusterData(eq("bar.googleapis.com"), clusterWatcherCaptor2.capture()); - - ClusterWatcher clusterWatcher2 = clusterWatcherCaptor2.getValue(); - clusterWatcher2.onClusterChanged( - ClusterUpdate.newBuilder() - .setClusterName("bar.googleapis.com") - .setEdsServiceName("edsServiceBar.googleapis.com") - .setLbPolicy("round_robin") - .setLrsServerName("lrsBar.googleapis.com") - .build()); + @Deprecated + @Override + public NameResolver.Factory getNameResolverFactory() { + throw new UnsupportedOperationException("should not be called"); + } - assertThat(edsLbHelpers).hasSize(1); - assertThat(edsLoadBalancers).hasSize(1); - Helper edsLbHelper2 = edsLbHelpers.poll(); - LoadBalancer edsLoadBalancer2 = edsLoadBalancers.poll(); - ArgumentCaptor resolvedAddressesCaptor2 = ArgumentCaptor.forClass(null); - verify(edsLoadBalancer2).handleResolvedAddresses(resolvedAddressesCaptor2.capture()); - expectedEdsConfig = new EdsConfig( - "bar.googleapis.com", - "edsServiceBar.googleapis.com", - "lrsBar.googleapis.com", - roundRobinPolicy); - ResolvedAddresses resolvedAddressesBar = resolvedAddressesCaptor2.getValue(); - assertThat(resolvedAddressesBar.getLoadBalancingPolicyConfig()).isEqualTo(expectedEdsConfig); - assertThat(resolvedAddressesBar.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL)) - .isSameInstanceAs(xdsClientPool); - - SubchannelPicker picker2 = mock(SubchannelPicker.class); - edsLbHelper2.updateBalancingState(ConnectivityState.CONNECTING, picker2); - verify(helper, never()).updateBalancingState(ConnectivityState.CONNECTING, picker2); - verify(edsLoadBalancer1, never()).shutdown(); - - picker2 = mock(SubchannelPicker.class); - edsLbHelper2.updateBalancingState(ConnectivityState.READY, picker2); - verify(helper).updateBalancingState(ConnectivityState.READY, picker2); - verify(edsLoadBalancer1).shutdown(); - verify(xdsClient).cancelClusterDataWatch("foo.googleapis.com", clusterWatcher1); - - clusterWatcher2.onClusterChanged( - ClusterUpdate.newBuilder() - .setClusterName("bar.googleapis.com") - .setEdsServiceName("edsServiceBar2.googleapis.com") - .setLbPolicy("round_robin") - .build()); - verify(edsLoadBalancer2, times(2)).handleResolvedAddresses(resolvedAddressesCaptor2.capture()); - expectedEdsConfig = new EdsConfig( - "bar.googleapis.com", - "edsServiceBar2.googleapis.com", - null, - roundRobinPolicy); - ResolvedAddresses resolvedAddressesBar2 = resolvedAddressesCaptor2.getValue(); - assertThat(resolvedAddressesBar2.getLoadBalancingPolicyConfig()).isEqualTo(expectedEdsConfig); - - cdsLoadBalancer.shutdown(); - verify(edsLoadBalancer2).shutdown(); - verify(xdsClient).cancelClusterDataWatch("bar.googleapis.com", clusterWatcher2); - assertThat(xdsClientPool.xdsClient).isNull(); + @Override + public String getAuthority() { + return AUTHORITY; + } } - @Test - public void handleCdsConfigUpdate_withUpstreamTlsContext() { - assertThat(cdsLoadBalancer).isInstanceOf(CdsLoadBalancer.class); - ((CdsLoadBalancer)cdsLoadBalancer).setXdsSecurity(true); - assertThat(xdsClient).isNull(); - ResolvedAddresses resolvedAddresses1 = - ResolvedAddresses.newBuilder() - .setAddresses(ImmutableList.of()) - .setAttributes( - Attributes.newBuilder() - .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) - .build()) - .setLoadBalancingPolicyConfig(new CdsConfig("foo.googleapis.com")) - .build(); - cdsLoadBalancer.handleResolvedAddresses(resolvedAddresses1); - - ArgumentCaptor clusterWatcherCaptor1 = ArgumentCaptor.forClass(null); - verify(xdsClient).watchClusterData(eq("foo.googleapis.com"), clusterWatcherCaptor1.capture()); + private static final class FakeSubchannel extends Subchannel { + private final List eags; - UpstreamTlsContext upstreamTlsContext = - CommonTlsContextTestsUtil.buildUpstreamTlsContextFromFilenames( - CLIENT_KEY_FILE, CLIENT_PEM_FILE, CA_PEM_FILE); - - SslContextProvider mockSslContextProvider = mock(SslContextProvider.class); - doReturn(upstreamTlsContext).when(mockSslContextProvider).getUpstreamTlsContext(); - doReturn(mockSslContextProvider).when(mockTlsContextManager) - .findOrCreateClientSslContextProvider(same(upstreamTlsContext)); - - ClusterWatcher clusterWatcher1 = clusterWatcherCaptor1.getValue(); - clusterWatcher1.onClusterChanged( - ClusterUpdate.newBuilder() - .setClusterName("foo.googleapis.com") - .setEdsServiceName("edsServiceFoo.googleapis.com") - .setLbPolicy("round_robin") - .setUpstreamTlsContext(upstreamTlsContext) - .build()); + private FakeSubchannel(List eags) { + this.eags = eags; + } - assertThat(edsLbHelpers).hasSize(1); - assertThat(edsLoadBalancers).hasSize(1); - verify(mockTlsContextManager, never()) - .releaseClientSslContextProvider(any(SslContextProvider.class)); - Helper edsLbHelper1 = edsLbHelpers.poll(); - - ArrayList eagList = new ArrayList<>(); - eagList.add(new EquivalentAddressGroup(new InetSocketAddress("foo.com", 8080))); - eagList.add(new EquivalentAddressGroup(InetSocketAddress.createUnresolved("localhost", 8081), - Attributes.newBuilder().set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool).build())); - LoadBalancer.CreateSubchannelArgs createSubchannelArgs = - LoadBalancer.CreateSubchannelArgs.newBuilder() - .setAddresses(eagList) - .build(); - ArgumentCaptor createSubchannelArgsCaptor1 = - ArgumentCaptor.forClass(null); - verify(helper, never()) - .createSubchannel(any(LoadBalancer.CreateSubchannelArgs.class)); - edsLbHelper1.createSubchannel(createSubchannelArgs); - verifyUpstreamTlsContextAttribute(upstreamTlsContext, - createSubchannelArgsCaptor1); - - // update with same upstreamTlsContext - reset(mockTlsContextManager); - clusterWatcher1.onClusterChanged( - ClusterUpdate.newBuilder() - .setClusterName("bar.googleapis.com") - .setEdsServiceName("eds1ServiceFoo.googleapis.com") - .setLbPolicy("round_robin") - .setUpstreamTlsContext(upstreamTlsContext) - .build()); + @Override + public void shutdown() { + } - verify(mockTlsContextManager, never()) - .releaseClientSslContextProvider(any(SslContextProvider.class)); - verify(mockTlsContextManager, never()).findOrCreateClientSslContextProvider( - any(UpstreamTlsContext.class)); + @Override + public void requestConnection() { + } - // update with different upstreamTlsContext - reset(mockTlsContextManager); - reset(helper); - UpstreamTlsContext upstreamTlsContext1 = - CommonTlsContextTestsUtil.buildUpstreamTlsContextFromFilenames( - BAD_CLIENT_KEY_FILE, BAD_CLIENT_PEM_FILE, CA_PEM_FILE); - SslContextProvider mockSslContextProvider1 = mock(SslContextProvider.class); - doReturn(upstreamTlsContext1).when(mockSslContextProvider1).getUpstreamTlsContext(); - doReturn(mockSslContextProvider1).when(mockTlsContextManager) - .findOrCreateClientSslContextProvider(same(upstreamTlsContext1)); - clusterWatcher1.onClusterChanged( - ClusterUpdate.newBuilder() - .setClusterName("bar.googleapis.com") - .setEdsServiceName("eds1ServiceFoo.googleapis.com") - .setLbPolicy("round_robin") - .setUpstreamTlsContext(upstreamTlsContext1) - .build()); + @Override + public List getAllAddresses() { + return eags; + } - verify(mockTlsContextManager).releaseClientSslContextProvider(same(mockSslContextProvider)); - verify(mockTlsContextManager).findOrCreateClientSslContextProvider(same(upstreamTlsContext1)); - ArgumentCaptor createSubchannelArgsCaptor2 = - ArgumentCaptor.forClass(null); - edsLbHelper1.createSubchannel(createSubchannelArgs); - verifyUpstreamTlsContextAttribute(upstreamTlsContext1, - createSubchannelArgsCaptor2); - - // update with null - reset(mockTlsContextManager); - reset(helper); - clusterWatcher1.onClusterChanged( - ClusterUpdate.newBuilder() - .setClusterName("bar.googleapis.com") - .setEdsServiceName("eds1ServiceFoo.googleapis.com") - .setLbPolicy("round_robin") - .setUpstreamTlsContext(null) - .build()); - verify(mockTlsContextManager).releaseClientSslContextProvider(same(mockSslContextProvider1)); - verify(mockTlsContextManager, never()).findOrCreateClientSslContextProvider( - any(UpstreamTlsContext.class)); - ArgumentCaptor createSubchannelArgsCaptor3 = - ArgumentCaptor.forClass(null); - edsLbHelper1.createSubchannel(createSubchannelArgs); - verifyUpstreamTlsContextAttribute(null, - createSubchannelArgsCaptor3); - - LoadBalancer edsLoadBalancer1 = edsLoadBalancers.poll(); - - cdsLoadBalancer.shutdown(); - verify(edsLoadBalancer1).shutdown(); - verify(xdsClient).cancelClusterDataWatch("foo.googleapis.com", clusterWatcher1); - assertThat(xdsClientPool.xdsClient).isNull(); + @Override + public Attributes getAttributes() { + return Attributes.EMPTY; + } } - private void verifyUpstreamTlsContextAttribute( - UpstreamTlsContext upstreamTlsContext, - ArgumentCaptor createSubchannelArgsCaptor1) { - verify(helper, times(1)).createSubchannel(createSubchannelArgsCaptor1.capture()); - CreateSubchannelArgs capturedValue = createSubchannelArgsCaptor1.getValue(); - List capturedEagList = capturedValue.getAddresses(); - assertThat(capturedEagList.size()).isEqualTo(2); - EquivalentAddressGroup capturedEag = capturedEagList.get(0); - UpstreamTlsContext capturedUpstreamTlsContext = - capturedEag.getAttributes().get(XdsAttributes.ATTR_UPSTREAM_TLS_CONTEXT); - assertThat(capturedUpstreamTlsContext).isSameInstanceAs(upstreamTlsContext); - capturedEag = capturedEagList.get(1); - capturedUpstreamTlsContext = - capturedEag.getAttributes().get(XdsAttributes.ATTR_UPSTREAM_TLS_CONTEXT); - assertThat(capturedUpstreamTlsContext).isSameInstanceAs(upstreamTlsContext); - assertThat(capturedEag.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL)) - .isSameInstanceAs(xdsClientPool); - } + private static final class FakeTlsContextManager implements TlsContextManager { - @Test - public void clusterWatcher_resourceNotExist() { - ResolvedAddresses resolvedAddresses = ResolvedAddresses.newBuilder() - .setAddresses(ImmutableList.of()) - .setAttributes(Attributes.newBuilder() - .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) - .build()) - .setLoadBalancingPolicyConfig(new CdsConfig("foo.googleapis.com")) - .build(); - cdsLoadBalancer.handleResolvedAddresses(resolvedAddresses); - - ArgumentCaptor clusterWatcherCaptor = ArgumentCaptor.forClass(null); - verify(xdsClient).watchClusterData(eq("foo.googleapis.com"), clusterWatcherCaptor.capture()); - - ClusterWatcher clusterWatcher = clusterWatcherCaptor.getValue(); - ArgumentCaptor pickerCaptor = ArgumentCaptor.forClass(null); - clusterWatcher.onResourceDoesNotExist("foo.googleapis.com"); - assertThat(edsLoadBalancers).isEmpty(); - verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); - PickResult result = pickerCaptor.getValue().pickSubchannel(mock(PickSubchannelArgs.class)); - assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAVAILABLE); - assertThat(result.getStatus().getDescription()) - .isEqualTo("Resource foo.googleapis.com is unavailable"); - } + @Override + public SslContextProvider findOrCreateClientSslContextProvider( + UpstreamTlsContext upstreamTlsContext) { + SslContextProvider sslContextProvider = mock(SslContextProvider.class); + when(sslContextProvider.getUpstreamTlsContext()).thenReturn(upstreamTlsContext); + return sslContextProvider; + } - @Test - public void clusterWatcher_resourceRemoved() { - ResolvedAddresses resolvedAddresses = ResolvedAddresses.newBuilder() - .setAddresses(ImmutableList.of()) - .setAttributes(Attributes.newBuilder() - .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) - .build()) - .setLoadBalancingPolicyConfig(new CdsConfig("foo.googleapis.com")) - .build(); - cdsLoadBalancer.handleResolvedAddresses(resolvedAddresses); - - ArgumentCaptor clusterWatcherCaptor = ArgumentCaptor.forClass(null); - verify(xdsClient).watchClusterData(eq("foo.googleapis.com"), clusterWatcherCaptor.capture()); - - ClusterWatcher clusterWatcher = clusterWatcherCaptor.getValue(); - ArgumentCaptor pickerCaptor = ArgumentCaptor.forClass(null); - clusterWatcher.onClusterChanged( - ClusterUpdate.newBuilder() - .setClusterName("foo.googleapis.com") - .setEdsServiceName("edsServiceFoo.googleapis.com") - .setLbPolicy("round_robin") - .build()); - assertThat(edsLoadBalancers).hasSize(1); - assertThat(edsLbHelpers).hasSize(1); - LoadBalancer edsLoadBalancer = edsLoadBalancers.poll(); - Helper edsHelper = edsLbHelpers.poll(); - SubchannelPicker subchannelPicker = mock(SubchannelPicker.class); - edsHelper.updateBalancingState(READY, subchannelPicker); - verify(helper).updateBalancingState(eq(READY), same(subchannelPicker)); - - clusterWatcher.onResourceDoesNotExist("foo.googleapis.com"); - verify(edsLoadBalancer).shutdown(); - verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); - PickResult result = pickerCaptor.getValue().pickSubchannel(mock(PickSubchannelArgs.class)); - assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAVAILABLE); - assertThat(result.getStatus().getDescription()) - .isEqualTo("Resource foo.googleapis.com is unavailable"); - } + @Override + public SslContextProvider releaseClientSslContextProvider( + SslContextProvider sslContextProvider) { + // no-op + return null; + } - @Test - public void clusterWatcher_onErrorCalledBeforeAndAfterOnClusterChanged() { - ResolvedAddresses resolvedAddresses = ResolvedAddresses.newBuilder() - .setAddresses(ImmutableList.of()) - .setAttributes(Attributes.newBuilder() - .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) - .build()) - .setLoadBalancingPolicyConfig(new CdsConfig("foo.googleapis.com")) - .build(); - cdsLoadBalancer.handleResolvedAddresses(resolvedAddresses); - - ArgumentCaptor clusterWatcherCaptor = ArgumentCaptor.forClass(null); - verify(xdsClient).watchClusterData(eq("foo.googleapis.com"), clusterWatcherCaptor.capture()); - - ClusterWatcher clusterWatcher = clusterWatcherCaptor.getValue(); - - // Call onError() before onClusterChanged() ever called. - clusterWatcher.onError(Status.DATA_LOSS.withDescription("fake status")); - assertThat(edsLoadBalancers).isEmpty(); - verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); - - clusterWatcher.onClusterChanged( - ClusterUpdate.newBuilder() - .setClusterName("foo.googleapis.com") - .setEdsServiceName("edsServiceFoo.googleapis.com") - .setLbPolicy("round_robin") - .build()); + @Override + public SslContextProvider findOrCreateServerSslContextProvider( + DownstreamTlsContext downstreamTlsContext) { + throw new UnsupportedOperationException("should not be called"); + } - assertThat(edsLbHelpers).hasSize(1); - assertThat(edsLoadBalancers).hasSize(1); - Helper edsLbHelper = edsLbHelpers.poll(); - LoadBalancer edsLoadBalancer = edsLoadBalancers.poll(); - verify(edsLoadBalancer).handleResolvedAddresses(any(ResolvedAddresses.class)); - SubchannelPicker picker = mock(SubchannelPicker.class); - - edsLbHelper.updateBalancingState(ConnectivityState.READY, picker); - verify(helper).updateBalancingState(ConnectivityState.READY, picker); - - // Call onError() after onClusterChanged(). - clusterWatcher.onError(Status.DATA_LOSS.withDescription("fake status")); - // Verify no more TRANSIENT_FAILURE. - verify(helper, times(1)) - .updateBalancingState(eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); + @Override + public SslContextProvider releaseServerSslContextProvider( + SslContextProvider sslContextProvider) { + throw new UnsupportedOperationException("should not be called"); + } } } From fa103b9d7a2fdf5806dc55845ef424061dcb4d99 Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Mon, 31 Aug 2020 10:27:48 -0700 Subject: [PATCH 08/86] Revert "xds: eliminate cluster name change logic in CDS LB policy and reimplement tests (#7356)" (#7379) This reverts commit 1260db3305074e9537513be48e275bd880e0277a. --- .../java/io/grpc/xds/CdsLoadBalancer.java | 193 ++-- .../java/io/grpc/xds/CdsLoadBalancerTest.java | 888 ++++++++++-------- 2 files changed, 613 insertions(+), 468 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java index cf9e7991df6..d0b9248ac42 100644 --- a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java @@ -32,6 +32,7 @@ import io.grpc.internal.ObjectPool; import io.grpc.internal.ServiceConfigUtil.PolicySelection; import io.grpc.util.ForwardingLoadBalancerHelper; +import io.grpc.util.GracefulSwitchLoadBalancer; import io.grpc.xds.CdsLoadBalancerProvider.CdsConfig; import io.grpc.xds.EdsLoadBalancerProvider.EdsConfig; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; @@ -44,24 +45,31 @@ import io.grpc.xds.internal.sds.TlsContextManagerImpl; import java.util.ArrayList; import java.util.List; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nullable; /** - * Load balancer for cds_experimental LB policy. One instance per cluster. + * Load balancer for cds_experimental LB policy. */ -final class CdsLoadBalancer extends LoadBalancer { +public final class CdsLoadBalancer extends LoadBalancer { private final XdsLogger logger; - private final Helper helper; private final LoadBalancerRegistry lbRegistry; + private final GracefulSwitchLoadBalancer switchingLoadBalancer; private final TlsContextManager tlsContextManager; // TODO(sanjaypujare): remove once xds security is released private boolean enableXdsSecurity; private static final String XDS_SECURITY_ENV_VAR = "GRPC_XDS_EXPERIMENTAL_SECURITY_SUPPORT"; + + // The following fields become non-null once handleResolvedAddresses() successfully. + + // Most recent cluster name. + @Nullable private String clusterName; + @Nullable private ObjectPool xdsClientPool; + @Nullable private XdsClient xdsClient; - private ChildLbState childLbState; - private ResolvedAddresses resolvedAddresses; CdsLoadBalancer(Helper helper) { this(helper, LoadBalancerRegistry.getDefaultRegistry(), TlsContextManagerImpl.getInstance()); @@ -70,8 +78,9 @@ final class CdsLoadBalancer extends LoadBalancer { @VisibleForTesting CdsLoadBalancer(Helper helper, LoadBalancerRegistry lbRegistry, TlsContextManager tlsContextManager) { - this.helper = checkNotNull(helper, "helper"); + checkNotNull(helper, "helper"); this.lbRegistry = lbRegistry; + this.switchingLoadBalancer = new GracefulSwitchLoadBalancer(helper); this.tlsContextManager = tlsContextManager; logger = XdsLogger.withLogId(InternalLogId.allocate("cds-lb", helper.getAuthority())); logger.log(XdsLogLevel.INFO, "Created"); @@ -79,32 +88,33 @@ final class CdsLoadBalancer extends LoadBalancer { @Override public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { - if (clusterName != null) { - return; - } logger.log(XdsLogLevel.DEBUG, "Received resolution result: {0}", resolvedAddresses); - this.resolvedAddresses = resolvedAddresses; - xdsClientPool = resolvedAddresses.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL); - checkNotNull(xdsClientPool, "missing xDS client pool"); - xdsClient = xdsClientPool.getObject(); + if (xdsClientPool == null) { + xdsClientPool = resolvedAddresses.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL); + checkNotNull(xdsClientPool, "missing xDS client pool"); + xdsClient = xdsClientPool.getObject(); + } + Object lbConfig = resolvedAddresses.getLoadBalancingPolicyConfig(); checkNotNull(lbConfig, "missing CDS lb config"); CdsConfig newCdsConfig = (CdsConfig) lbConfig; logger.log( XdsLogLevel.INFO, "Received CDS lb config: cluster={0}", newCdsConfig.name); + + // If cluster is changed, do a graceful switch. + if (!newCdsConfig.name.equals(clusterName)) { + LoadBalancer.Factory clusterBalancerFactory = new ClusterBalancerFactory(newCdsConfig.name); + switchingLoadBalancer.switchTo(clusterBalancerFactory); + } + switchingLoadBalancer.handleResolvedAddresses(resolvedAddresses); clusterName = newCdsConfig.name; - childLbState = new ChildLbState(); } @Override public void handleNameResolutionError(Status error) { logger.log(XdsLogLevel.WARNING, "Received name resolution error: {0}", error); - if (childLbState != null) { - childLbState.propagateError(error); - } else { - helper.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(error)); - } + switchingLoadBalancer.handleNameResolutionError(error); } @Override @@ -115,9 +125,7 @@ public boolean canHandleEmptyAddressListFromNameResolution() { @Override public void shutdown() { logger.log(XdsLogLevel.INFO, "Shutdown"); - if (childLbState != null) { - childLbState.shutdown(); - } + switchingLoadBalancer.shutdown(); if (xdsClientPool != null) { xdsClientPool.returnObject(xdsClient); } @@ -134,25 +142,107 @@ void setXdsSecurity(boolean enable) { enableXdsSecurity = enable; } - private final class ChannelSecurityLbHelper extends ForwardingLoadBalancerHelper { - @Nullable - private SslContextProvider sslContextProvider; + /** + * A load balancer factory that provides a load balancer for a given cluster. + */ + private final class ClusterBalancerFactory extends LoadBalancer.Factory { + + final String clusterName; + + ClusterBalancerFactory(String clusterName) { + this.clusterName = clusterName; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof ClusterBalancerFactory)) { + return false; + } + ClusterBalancerFactory that = (ClusterBalancerFactory) o; + return clusterName.equals(that.clusterName); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), clusterName); + } + + @Override + public LoadBalancer newLoadBalancer(final Helper helper) { + return new LoadBalancer() { + // Becomes non-null once handleResolvedAddresses() successfully. + // Assigned at most once. + @Nullable + ClusterWatcherImpl clusterWatcher; + + @Override + public void handleNameResolutionError(Status error) { + if (clusterWatcher == null || clusterWatcher.edsBalancer == null) { + // Go into TRANSIENT_FAILURE if we have not yet received any cluster resource. + // Otherwise, we keep running with the data we had previously. + helper.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(error)); + } + } + + @Override + public boolean canHandleEmptyAddressListFromNameResolution() { + return true; + } + + @Override + public void shutdown() { + if (clusterWatcher != null) { + if (clusterWatcher.edsBalancer != null) { + clusterWatcher.edsBalancer.shutdown(); + } + xdsClient.cancelClusterDataWatch(clusterName, clusterWatcher); + logger.log( + XdsLogLevel.INFO, + "Cancelled cluster watcher on {0} with xDS client {1}", + clusterName, xdsClient); + } + } + + @Override + public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { + if (clusterWatcher == null) { + clusterWatcher = new ClusterWatcherImpl(helper, resolvedAddresses); + logger.log( + XdsLogLevel.INFO, + "Start cluster watcher on {0} with xDS client {1}", + clusterName, xdsClient); + xdsClient.watchClusterData(clusterName, clusterWatcher); + } + } + }; + } + } + + private static final class EdsLoadBalancingHelper extends ForwardingLoadBalancerHelper { + private final Helper delegate; + private final AtomicReference sslContextProvider; + + EdsLoadBalancingHelper(Helper helper, + AtomicReference sslContextProvider) { + this.delegate = helper; + this.sslContextProvider = sslContextProvider; + } @Override public Subchannel createSubchannel(CreateSubchannelArgs createSubchannelArgs) { - if (sslContextProvider != null) { + if (sslContextProvider.get() != null) { createSubchannelArgs = createSubchannelArgs .toBuilder() .setAddresses( addUpstreamTlsContext(createSubchannelArgs.getAddresses(), - sslContextProvider.getUpstreamTlsContext())) + sslContextProvider.get().getUpstreamTlsContext())) .build(); } - return delegate().createSubchannel(createSubchannelArgs); + return delegate.createSubchannel(createSubchannelArgs); } - private List addUpstreamTlsContext( + private static List addUpstreamTlsContext( List addresses, UpstreamTlsContext upstreamTlsContext) { if (upstreamTlsContext == null || addresses == null) { @@ -174,19 +264,22 @@ private List addUpstreamTlsContext( @Override protected Helper delegate() { - return helper; + return delegate; } } - private final class ChildLbState implements ClusterWatcher { - private final ChannelSecurityLbHelper lbHelper = new ChannelSecurityLbHelper(); + private final class ClusterWatcherImpl implements ClusterWatcher { + + final EdsLoadBalancingHelper helper; + final ResolvedAddresses resolvedAddresses; + @Nullable LoadBalancer edsBalancer; - private ChildLbState() { - xdsClient.watchClusterData(clusterName, this); - logger.log(XdsLogLevel.INFO, - "Started watcher for cluster {0} with xDS client {1}", clusterName, xdsClient); + ClusterWatcherImpl(Helper helper, ResolvedAddresses resolvedAddresses) { + this.helper = new EdsLoadBalancingHelper(helper, + new AtomicReference()); + this.resolvedAddresses = resolvedAddresses; } @Override @@ -215,7 +308,7 @@ public void onClusterChanged(ClusterUpdate newUpdate) { updateSslContextProvider(newUpdate.getUpstreamTlsContext()); } if (edsBalancer == null) { - edsBalancer = lbRegistry.getProvider(EDS_POLICY_NAME).newLoadBalancer(lbHelper); + edsBalancer = lbRegistry.getProvider(EDS_POLICY_NAME).newLoadBalancer(helper); } edsBalancer.handleResolvedAddresses( resolvedAddresses.toBuilder().setLoadBalancingPolicyConfig(edsConfig).build()); @@ -223,7 +316,8 @@ public void onClusterChanged(ClusterUpdate newUpdate) { /** For new UpstreamTlsContext value, release old SslContextProvider. */ private void updateSslContextProvider(UpstreamTlsContext newUpstreamTlsContext) { - SslContextProvider oldSslContextProvider = lbHelper.sslContextProvider; + SslContextProvider oldSslContextProvider = + helper.sslContextProvider.get(); if (oldSslContextProvider != null) { UpstreamTlsContext oldUpstreamTlsContext = oldSslContextProvider.getUpstreamTlsContext(); @@ -233,10 +327,11 @@ private void updateSslContextProvider(UpstreamTlsContext newUpstreamTlsContext) tlsContextManager.releaseClientSslContextProvider(oldSslContextProvider); } if (newUpstreamTlsContext != null) { - lbHelper.sslContextProvider = + SslContextProvider newSslContextProvider = tlsContextManager.findOrCreateClientSslContextProvider(newUpstreamTlsContext); + helper.sslContextProvider.set(newSslContextProvider); } else { - lbHelper.sslContextProvider = null; + helper.sslContextProvider.set(null); } } @@ -265,22 +360,6 @@ public void onError(Status error) { helper.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(error)); } } - - void shutdown() { - xdsClient.cancelClusterDataWatch(clusterName, this); - logger.log(XdsLogLevel.INFO, - "Cancelled watcher for cluster {0} with xDS client {1}", clusterName, xdsClient); - if (edsBalancer != null) { - edsBalancer.shutdown(); - } - } - - void propagateError(Status error) { - if (edsBalancer != null) { - edsBalancer.handleNameResolutionError(error); - } else { - helper.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(error)); - } - } } + } diff --git a/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java index 1f2261cbdef..9a9bb69241f 100644 --- a/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java @@ -17,10 +17,26 @@ package io.grpc.xds; import static com.google.common.truth.Truth.assertThat; +import static io.grpc.ConnectivityState.CONNECTING; +import static io.grpc.ConnectivityState.READY; +import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; +import static io.grpc.xds.XdsLbPolicies.EDS_POLICY_NAME; +import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.BAD_CLIENT_KEY_FILE; +import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.BAD_CLIENT_PEM_FILE; +import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.CA_PEM_FILE; +import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.CLIENT_KEY_FILE; +import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.CLIENT_PEM_FILE; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; -import com.google.common.collect.Iterables; +import com.google.common.collect.ImmutableList; import io.grpc.Attributes; import io.grpc.ConnectivityState; import io.grpc.EquivalentAddressGroup; @@ -30,34 +46,38 @@ import io.grpc.LoadBalancer.PickResult; import io.grpc.LoadBalancer.PickSubchannelArgs; import io.grpc.LoadBalancer.ResolvedAddresses; -import io.grpc.LoadBalancer.Subchannel; import io.grpc.LoadBalancer.SubchannelPicker; import io.grpc.LoadBalancerProvider; import io.grpc.LoadBalancerRegistry; -import io.grpc.ManagedChannel; -import io.grpc.NameResolver; +import io.grpc.NameResolver.ConfigOrError; import io.grpc.Status; import io.grpc.Status.Code; import io.grpc.SynchronizationContext; -import io.grpc.internal.ObjectPool; +import io.grpc.internal.FakeClock; +import io.grpc.internal.ServiceConfigUtil.PolicySelection; import io.grpc.xds.CdsLoadBalancerProvider.CdsConfig; import io.grpc.xds.EdsLoadBalancerProvider.EdsConfig; -import io.grpc.xds.EnvoyServerProtoData.DownstreamTlsContext; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; +import io.grpc.xds.XdsClient.ClusterUpdate; +import io.grpc.xds.XdsClient.ClusterWatcher; +import io.grpc.xds.XdsClient.RefCountedXdsClientObjectPool; +import io.grpc.xds.XdsClient.XdsClientFactory; import io.grpc.xds.internal.sds.CommonTlsContextTestsUtil; import io.grpc.xds.internal.sds.SslContextProvider; import io.grpc.xds.internal.sds.TlsContextManager; -import java.net.SocketAddress; +import java.net.InetSocketAddress; +import java.util.ArrayDeque; import java.util.ArrayList; -import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; import java.util.List; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import org.junit.After; +import java.util.Map; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; import org.mockito.MockitoAnnotations; /** @@ -65,323 +85,44 @@ */ @RunWith(JUnit4.class) public class CdsLoadBalancerTest { - private static final String AUTHORITY = "api.google.com"; - private static final String CLUSTER = "cluster-foo.googleapis.com"; - private final SynchronizationContext syncContext = new SynchronizationContext( - new Thread.UncaughtExceptionHandler() { + + private final RefCountedXdsClientObjectPool xdsClientPool = new RefCountedXdsClientObjectPool( + new XdsClientFactory() { @Override - public void uncaughtException(Thread t, Throwable e) { - throw new AssertionError(e); + XdsClient createXdsClient() { + xdsClient = mock(XdsClient.class); + return xdsClient; } - }); - private final List childBalancers = new ArrayList<>(); - private final FakeXdsClient xdsClient = new FakeXdsClient(); - private final TlsContextManager tlsContextManager = new FakeTlsContextManager(); - private LoadBalancer.Helper helper = new FakeLbHelper(); - private int xdsClientRefs; - private ConnectivityState currentState; - private SubchannelPicker currentPicker; - private CdsLoadBalancer loadBalancer; - - @Before - public void setUp() { - MockitoAnnotations.initMocks(this); - - LoadBalancerRegistry registry = new LoadBalancerRegistry(); - registry.register(new FakeLoadBalancerProvider(XdsLbPolicies.EDS_POLICY_NAME)); - registry.register(new FakeLoadBalancerProvider("round_robin")); - ObjectPool xdsClientPool = new ObjectPool() { - @Override - public XdsClient getObject() { - xdsClientRefs++; - return xdsClient; } + ); - @Override - public XdsClient returnObject(Object object) { - assertThat(xdsClientRefs).isGreaterThan(0); - xdsClientRefs--; - return null; - } - }; - loadBalancer = new CdsLoadBalancer(helper, registry, tlsContextManager); - loadBalancer.handleResolvedAddresses( - ResolvedAddresses.newBuilder() - .setAddresses(Collections.emptyList()) - .setLoadBalancingPolicyConfig(new CdsConfig(CLUSTER)) - .setAttributes( - Attributes.newBuilder().set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool).build()) - .build()); - assertThat(xdsClient.watcher).isNotNull(); - } - - @After - public void tearDown() { - loadBalancer.shutdown(); - assertThat(xdsClient.watcher).isNull(); - assertThat(xdsClientRefs).isEqualTo(0); - } - - - @Test - public void receiveFirstClusterResourceInfo() { - xdsClient.deliverClusterInfo(null, null); - assertThat(childBalancers).hasSize(1); - FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); - assertThat(childBalancer.name).isEqualTo(XdsLbPolicies.EDS_POLICY_NAME); - assertThat(childBalancer.config).isNotNull(); - EdsConfig edsConfig = (EdsConfig) childBalancer.config; - assertThat(edsConfig.clusterName).isEqualTo(CLUSTER); - assertThat(edsConfig.edsServiceName).isNull(); - assertThat(edsConfig.lrsServerName).isNull(); - assertThat(edsConfig.endpointPickingPolicy.getProvider().getPolicyName()) - .isEqualTo("round_robin"); - } - - @Test - public void clusterResourceNeverExist() { - xdsClient.deliverResourceNotFound(); - assertThat(childBalancers).isEmpty(); - assertThat(currentState).isEqualTo(ConnectivityState.TRANSIENT_FAILURE); - PickResult result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); - assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAVAILABLE); - assertThat(result.getStatus().getDescription()) - .isEqualTo("Resource " + CLUSTER + " is unavailable"); - } - - @Test - public void clusterResourceRemoved() { - xdsClient.deliverClusterInfo(null, null); - assertThat(childBalancers).hasSize(1); - FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); - assertThat(childBalancer.shutdown).isFalse(); - - xdsClient.deliverResourceNotFound(); - assertThat(childBalancer.shutdown).isTrue(); - assertThat(currentState).isEqualTo(ConnectivityState.TRANSIENT_FAILURE); - PickResult result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); - assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAVAILABLE); - assertThat(result.getStatus().getDescription()) - .isEqualTo("Resource " + CLUSTER + " is unavailable"); - } - - @Test - public void clusterResourceUpdated() { - xdsClient.deliverClusterInfo(null, null); - FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); - EdsConfig edsConfig = (EdsConfig) childBalancer.config; - assertThat(edsConfig.clusterName).isEqualTo(CLUSTER); - assertThat(edsConfig.edsServiceName).isNull(); - assertThat(edsConfig.lrsServerName).isNull(); - assertThat(edsConfig.endpointPickingPolicy.getProvider().getPolicyName()) - .isEqualTo("round_robin"); - - String edsService = "service-bar.googleapis.com"; - String loadReportServer = "lrs-server.googleapis.com"; - xdsClient.deliverClusterInfo(edsService, loadReportServer); - assertThat(childBalancers).containsExactly(childBalancer); - edsConfig = (EdsConfig) childBalancer.config; - assertThat(edsConfig.clusterName).isEqualTo(CLUSTER); - assertThat(edsConfig.edsServiceName).isEqualTo(edsService); - assertThat(edsConfig.lrsServerName).isEqualTo(loadReportServer); - assertThat(edsConfig.endpointPickingPolicy.getProvider().getPolicyName()) - .isEqualTo("round_robin"); - } - - @Test - public void receiveClusterResourceInfoWithUpstreamTlsContext() { - loadBalancer.setXdsSecurity(true); - UpstreamTlsContext upstreamTlsContext = - CommonTlsContextTestsUtil.buildUpstreamTlsContextFromFilenames( - CommonTlsContextTestsUtil.CLIENT_KEY_FILE, - CommonTlsContextTestsUtil.CLIENT_PEM_FILE, - CommonTlsContextTestsUtil.CA_PEM_FILE); - xdsClient.deliverClusterInfo(null, null, upstreamTlsContext); - FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); - List addresses = createEndpointAddresses(2); - CreateSubchannelArgs args = - CreateSubchannelArgs.newBuilder() - .setAddresses(addresses) - .build(); - Subchannel subchannel = childBalancer.helper.createSubchannel(args); - for (EquivalentAddressGroup eag : subchannel.getAllAddresses()) { - assertThat(eag.getAttributes().get(XdsAttributes.ATTR_UPSTREAM_TLS_CONTEXT)) - .isEqualTo(upstreamTlsContext); - } - - xdsClient.deliverClusterInfo(null, null); - subchannel = childBalancer.helper.createSubchannel(args); - for (EquivalentAddressGroup eag : subchannel.getAllAddresses()) { - assertThat(eag.getAttributes().get(XdsAttributes.ATTR_UPSTREAM_TLS_CONTEXT)).isNull(); - } - - upstreamTlsContext = - CommonTlsContextTestsUtil.buildUpstreamTlsContextFromFilenames( - CommonTlsContextTestsUtil.BAD_CLIENT_KEY_FILE, - CommonTlsContextTestsUtil.BAD_CLIENT_PEM_FILE, - CommonTlsContextTestsUtil.CA_PEM_FILE); - xdsClient.deliverClusterInfo(null, null, upstreamTlsContext); - subchannel = childBalancer.helper.createSubchannel(args); - for (EquivalentAddressGroup eag : subchannel.getAllAddresses()) { - assertThat(eag.getAttributes().get(XdsAttributes.ATTR_UPSTREAM_TLS_CONTEXT)) - .isEqualTo(upstreamTlsContext); - } - } - - @Test - public void subchannelStatePropagateFromDownstreamToUpstream() { - xdsClient.deliverClusterInfo(null, null); - FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); - List addresses = createEndpointAddresses(2); - CreateSubchannelArgs args = - CreateSubchannelArgs.newBuilder() - .setAddresses(addresses) - .build(); - Subchannel subchannel = childBalancer.helper.createSubchannel(args); - childBalancer.deliverSubchannelState(subchannel, ConnectivityState.READY); - assertThat(currentState).isEqualTo(ConnectivityState.READY); - assertThat(currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)).getSubchannel()) - .isSameInstanceAs(subchannel); - } - - @Test - public void clusterDiscoveryError_beforeChildPolicyInstantiated_propagateToUpstream() { - xdsClient.deliverError(Status.UNAUTHENTICATED.withDescription("permission denied")); - assertThat(currentState).isEqualTo(ConnectivityState.TRANSIENT_FAILURE); - PickResult result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); - assertThat(result.getStatus().isOk()).isFalse(); - assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAUTHENTICATED); - assertThat(result.getStatus().getDescription()).isEqualTo("permission denied"); - } - - @Test - public void clusterDiscoveryError_afterChildPolicyInstantiated_keepUsingCurrentCluster() { - xdsClient.deliverClusterInfo(null, null); - FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); - xdsClient.deliverError(Status.UNAVAILABLE.withDescription("unreachable")); - assertThat(currentState).isNull(); - assertThat(currentPicker).isNull(); - assertThat(childBalancer.shutdown).isFalse(); - } - - @Test - public void nameResolutionError_beforeChildPolicyInstantiated_returnErrorPickerToUpstream() { - loadBalancer.handleNameResolutionError( - Status.UNIMPLEMENTED.withDescription("not found")); - assertThat(currentState).isEqualTo(ConnectivityState.TRANSIENT_FAILURE); - PickResult result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); - assertThat(result.getStatus().isOk()).isFalse(); - assertThat(result.getStatus().getCode()).isEqualTo(Code.UNIMPLEMENTED); - assertThat(result.getStatus().getDescription()).isEqualTo("not found"); - } - - @Test - public void nameResolutionError_afterChildPolicyInstantiated_propagateToDownstream() { - xdsClient.deliverClusterInfo(null, null); - FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); - loadBalancer.handleNameResolutionError( - Status.UNAVAILABLE.withDescription("cannot reach server")); - assertThat(childBalancer.upstreamError.getCode()).isEqualTo(Code.UNAVAILABLE); - assertThat(childBalancer.upstreamError.getDescription()) - .isEqualTo("cannot reach server"); - } - - private static List createEndpointAddresses(int n) { - List list = new ArrayList<>(); - for (int i = 0; i < n; i++) { - list.add(new EquivalentAddressGroup(mock(SocketAddress.class))); - } - return list; - } - - private final class FakeXdsClient extends XdsClient { - private ClusterWatcher watcher; - + private final LoadBalancerRegistry lbRegistry = new LoadBalancerRegistry(); + private final LoadBalancerProvider fakeEdsLoadBlancerProvider = new LoadBalancerProvider() { @Override - void watchClusterData(String clusterName, ClusterWatcher watcher) { - assertThat(clusterName).isEqualTo(CLUSTER); - this.watcher = watcher; + public boolean isAvailable() { + return true; } @Override - void cancelClusterDataWatch(String clusterName, ClusterWatcher watcher) { - assertThat(clusterName).isEqualTo(CLUSTER); - assertThat(watcher).isSameInstanceAs(this.watcher); - this.watcher = null; + public int getPriority() { + return 5; } @Override - void shutdown() { - // no-op - } - - void deliverClusterInfo( - @Nullable final String edsServiceName, @Nullable final String lrsServerName) { - syncContext.execute(new Runnable() { - @Override - public void run() { - watcher.onClusterChanged( - ClusterUpdate.newBuilder() - .setClusterName(CLUSTER) - .setEdsServiceName(edsServiceName) - .setLbPolicy("round_robin") // only supported policy - .setLrsServerName(lrsServerName) - .build()); - } - }); - } - - void deliverClusterInfo( - @Nullable final String edsServiceName, @Nullable final String lrsServerName, - final UpstreamTlsContext tlsContext) { - syncContext.execute(new Runnable() { - @Override - public void run() { - watcher.onClusterChanged( - ClusterUpdate.newBuilder() - .setClusterName(CLUSTER) - .setEdsServiceName(edsServiceName) - .setLbPolicy("round_robin") // only supported policy - .setLrsServerName(lrsServerName) - .setUpstreamTlsContext(tlsContext) - .build()); - } - }); - } - - void deliverResourceNotFound() { - syncContext.execute(new Runnable() { - @Override - public void run() { - watcher.onResourceDoesNotExist(CLUSTER); - } - }); - } - - void deliverError(final Status error) { - syncContext.execute(new Runnable() { - @Override - public void run() { - watcher.onError(error); - } - }); - } - } - - private final class FakeLoadBalancerProvider extends LoadBalancerProvider { - private final String policyName; - - FakeLoadBalancerProvider(String policyName) { - this.policyName = policyName; + public String getPolicyName() { + return EDS_POLICY_NAME; } @Override public LoadBalancer newLoadBalancer(Helper helper) { - FakeLoadBalancer balancer = new FakeLoadBalancer(policyName, helper); - childBalancers.add(balancer); - return balancer; + edsLbHelpers.add(helper); + LoadBalancer edsLoadBalancer = mock(LoadBalancer.class); + edsLoadBalancers.add(edsLoadBalancer); + return edsLoadBalancer; } + }; + private final LoadBalancerProvider fakeRoundRobinLbProvider = new LoadBalancerProvider() { @Override public boolean isAvailable() { return true; @@ -389,138 +130,463 @@ public boolean isAvailable() { @Override public int getPriority() { - return 0; // doesn't matter + return 5; } @Override public String getPolicyName() { - return policyName; - } - } - - private final class FakeLoadBalancer extends LoadBalancer { - private final String name; - private final Helper helper; - private Object config; - private Status upstreamError; - private boolean shutdown; - - FakeLoadBalancer(String name, Helper helper) { - this.name = name; - this.helper = helper; + return "round_robin"; } @Override - public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { - config = resolvedAddresses.getLoadBalancingPolicyConfig(); - } - - @Override - public void handleNameResolutionError(Status error) { - upstreamError = error; + public LoadBalancer newLoadBalancer(Helper helper) { + return mock(LoadBalancer.class); } @Override - public void shutdown() { - shutdown = true; - childBalancers.remove(this); + public ConfigOrError parseLoadBalancingPolicyConfig( + Map rawLoadBalancingPolicyConfig) { + return ConfigOrError.fromConfig("fake round robin config"); } + }; - void deliverSubchannelState(final Subchannel subchannel, ConnectivityState state) { - SubchannelPicker picker = new SubchannelPicker() { + private final SynchronizationContext syncContext = new SynchronizationContext( + new Thread.UncaughtExceptionHandler() { @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withSubchannel(subchannel); + public void uncaughtException(Thread t, Throwable e) { + throw new AssertionError(e); } - }; - helper.updateBalancingState(state, picker); - } + }); + + private final FakeClock fakeClock = new FakeClock(); + private final Deque edsLoadBalancers = new ArrayDeque<>(); + private final Deque edsLbHelpers = new ArrayDeque<>(); + + @Mock + private Helper helper; + + private LoadBalancer cdsLoadBalancer; + private XdsClient xdsClient; + + @Mock + private TlsContextManager mockTlsContextManager; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + + doReturn(syncContext).when(helper).getSynchronizationContext(); + doReturn(fakeClock.getScheduledExecutorService()).when(helper).getScheduledExecutorService(); + lbRegistry.register(fakeEdsLoadBlancerProvider); + lbRegistry.register(fakeRoundRobinLbProvider); + cdsLoadBalancer = new CdsLoadBalancer(helper, lbRegistry, mockTlsContextManager); } - private final class FakeLbHelper extends LoadBalancer.Helper { + @Test + public void canHandleEmptyAddressListFromNameResolution() { + assertThat(cdsLoadBalancer.canHandleEmptyAddressListFromNameResolution()).isTrue(); + } - @Override - public void updateBalancingState( - @Nonnull ConnectivityState newState, @Nonnull SubchannelPicker newPicker) { - currentState = newState; - currentPicker = newPicker; - } + @Test + public void handleResolutionErrorBeforeOrAfterCdsWorking() { + ResolvedAddresses resolvedAddresses1 = ResolvedAddresses.newBuilder() + .setAddresses(ImmutableList.of()) + .setAttributes(Attributes.newBuilder() + .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) + .build()) + .setLoadBalancingPolicyConfig(new CdsConfig("foo.googleapis.com")) + .build(); + cdsLoadBalancer.handleResolvedAddresses(resolvedAddresses1); + ArgumentCaptor clusterWatcherCaptor1 = ArgumentCaptor.forClass(null); + verify(xdsClient).watchClusterData(eq("foo.googleapis.com"), clusterWatcherCaptor1.capture()); + ClusterWatcher clusterWatcher1 = clusterWatcherCaptor1.getValue(); + + // handleResolutionError() before receiving any CDS response. + cdsLoadBalancer.handleNameResolutionError(Status.DATA_LOSS.withDescription("fake status")); + verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); + + // CDS response received. + clusterWatcher1.onClusterChanged( + ClusterUpdate.newBuilder() + .setClusterName("foo.googleapis.com") + .setEdsServiceName("edsServiceFoo.googleapis.com") + .setLbPolicy("round_robin") + .build()); + verify(helper).updateBalancingState(eq(CONNECTING), any(SubchannelPicker.class)); - @Override - public Subchannel createSubchannel(CreateSubchannelArgs args) { - return new FakeSubchannel(args.getAddresses()); - } + // handleResolutionError() after receiving CDS response. + cdsLoadBalancer.handleNameResolutionError(Status.DATA_LOSS.withDescription("fake status")); + // No more TRANSIENT_FAILURE. + verify(helper, times(1)).updateBalancingState( + eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); + } - @Override - public ManagedChannel createOobChannel(EquivalentAddressGroup eag, String authority) { - throw new UnsupportedOperationException("should not be called"); - } + @Test + public void handleCdsConfigUpdate() { + assertThat(xdsClient).isNull(); + ResolvedAddresses resolvedAddresses1 = ResolvedAddresses.newBuilder() + .setAddresses(ImmutableList.of()) + .setAttributes(Attributes.newBuilder() + .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) + .build()) + .setLoadBalancingPolicyConfig(new CdsConfig("foo.googleapis.com")) + .build(); + cdsLoadBalancer.handleResolvedAddresses(resolvedAddresses1); + + ArgumentCaptor clusterWatcherCaptor1 = ArgumentCaptor.forClass(null); + verify(xdsClient).watchClusterData(eq("foo.googleapis.com"), clusterWatcherCaptor1.capture()); + + ClusterWatcher clusterWatcher1 = clusterWatcherCaptor1.getValue(); + clusterWatcher1.onClusterChanged( + ClusterUpdate.newBuilder() + .setClusterName("foo.googleapis.com") + .setEdsServiceName("edsServiceFoo.googleapis.com") + .setLbPolicy("round_robin") + .build()); - @Deprecated - @Override - public NameResolver.Factory getNameResolverFactory() { - throw new UnsupportedOperationException("should not be called"); - } + assertThat(edsLbHelpers).hasSize(1); + assertThat(edsLoadBalancers).hasSize(1); + Helper edsLbHelper1 = edsLbHelpers.poll(); + LoadBalancer edsLoadBalancer1 = edsLoadBalancers.poll(); + ArgumentCaptor resolvedAddressesCaptor1 = ArgumentCaptor.forClass(null); + verify(edsLoadBalancer1).handleResolvedAddresses(resolvedAddressesCaptor1.capture()); + PolicySelection roundRobinPolicy = new PolicySelection( + fakeRoundRobinLbProvider, new HashMap(), "fake round robin config"); + EdsConfig expectedEdsConfig = new EdsConfig( + "foo.googleapis.com", + "edsServiceFoo.googleapis.com", + null, + roundRobinPolicy); + ResolvedAddresses resolvedAddressesFoo = resolvedAddressesCaptor1.getValue(); + assertThat(resolvedAddressesFoo.getLoadBalancingPolicyConfig()).isEqualTo(expectedEdsConfig); + assertThat(resolvedAddressesFoo.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL)) + .isSameInstanceAs(xdsClientPool); + + SubchannelPicker picker1 = mock(SubchannelPicker.class); + edsLbHelper1.updateBalancingState(ConnectivityState.READY, picker1); + verify(helper).updateBalancingState(ConnectivityState.READY, picker1); + + ResolvedAddresses resolvedAddresses2 = ResolvedAddresses.newBuilder() + .setAddresses(ImmutableList.of()) + .setAttributes(Attributes.newBuilder() + .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) + .build()) + .setLoadBalancingPolicyConfig(new CdsConfig("bar.googleapis.com")) + .build(); + cdsLoadBalancer.handleResolvedAddresses(resolvedAddresses2); + + ArgumentCaptor clusterWatcherCaptor2 = ArgumentCaptor.forClass(null); + verify(xdsClient).watchClusterData(eq("bar.googleapis.com"), clusterWatcherCaptor2.capture()); + + ClusterWatcher clusterWatcher2 = clusterWatcherCaptor2.getValue(); + clusterWatcher2.onClusterChanged( + ClusterUpdate.newBuilder() + .setClusterName("bar.googleapis.com") + .setEdsServiceName("edsServiceBar.googleapis.com") + .setLbPolicy("round_robin") + .setLrsServerName("lrsBar.googleapis.com") + .build()); - @Override - public String getAuthority() { - return AUTHORITY; - } + assertThat(edsLbHelpers).hasSize(1); + assertThat(edsLoadBalancers).hasSize(1); + Helper edsLbHelper2 = edsLbHelpers.poll(); + LoadBalancer edsLoadBalancer2 = edsLoadBalancers.poll(); + ArgumentCaptor resolvedAddressesCaptor2 = ArgumentCaptor.forClass(null); + verify(edsLoadBalancer2).handleResolvedAddresses(resolvedAddressesCaptor2.capture()); + expectedEdsConfig = new EdsConfig( + "bar.googleapis.com", + "edsServiceBar.googleapis.com", + "lrsBar.googleapis.com", + roundRobinPolicy); + ResolvedAddresses resolvedAddressesBar = resolvedAddressesCaptor2.getValue(); + assertThat(resolvedAddressesBar.getLoadBalancingPolicyConfig()).isEqualTo(expectedEdsConfig); + assertThat(resolvedAddressesBar.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL)) + .isSameInstanceAs(xdsClientPool); + + SubchannelPicker picker2 = mock(SubchannelPicker.class); + edsLbHelper2.updateBalancingState(ConnectivityState.CONNECTING, picker2); + verify(helper, never()).updateBalancingState(ConnectivityState.CONNECTING, picker2); + verify(edsLoadBalancer1, never()).shutdown(); + + picker2 = mock(SubchannelPicker.class); + edsLbHelper2.updateBalancingState(ConnectivityState.READY, picker2); + verify(helper).updateBalancingState(ConnectivityState.READY, picker2); + verify(edsLoadBalancer1).shutdown(); + verify(xdsClient).cancelClusterDataWatch("foo.googleapis.com", clusterWatcher1); + + clusterWatcher2.onClusterChanged( + ClusterUpdate.newBuilder() + .setClusterName("bar.googleapis.com") + .setEdsServiceName("edsServiceBar2.googleapis.com") + .setLbPolicy("round_robin") + .build()); + verify(edsLoadBalancer2, times(2)).handleResolvedAddresses(resolvedAddressesCaptor2.capture()); + expectedEdsConfig = new EdsConfig( + "bar.googleapis.com", + "edsServiceBar2.googleapis.com", + null, + roundRobinPolicy); + ResolvedAddresses resolvedAddressesBar2 = resolvedAddressesCaptor2.getValue(); + assertThat(resolvedAddressesBar2.getLoadBalancingPolicyConfig()).isEqualTo(expectedEdsConfig); + + cdsLoadBalancer.shutdown(); + verify(edsLoadBalancer2).shutdown(); + verify(xdsClient).cancelClusterDataWatch("bar.googleapis.com", clusterWatcher2); + assertThat(xdsClientPool.xdsClient).isNull(); } - private static final class FakeSubchannel extends Subchannel { - private final List eags; + @Test + public void handleCdsConfigUpdate_withUpstreamTlsContext() { + assertThat(cdsLoadBalancer).isInstanceOf(CdsLoadBalancer.class); + ((CdsLoadBalancer)cdsLoadBalancer).setXdsSecurity(true); + assertThat(xdsClient).isNull(); + ResolvedAddresses resolvedAddresses1 = + ResolvedAddresses.newBuilder() + .setAddresses(ImmutableList.of()) + .setAttributes( + Attributes.newBuilder() + .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) + .build()) + .setLoadBalancingPolicyConfig(new CdsConfig("foo.googleapis.com")) + .build(); + cdsLoadBalancer.handleResolvedAddresses(resolvedAddresses1); + + ArgumentCaptor clusterWatcherCaptor1 = ArgumentCaptor.forClass(null); + verify(xdsClient).watchClusterData(eq("foo.googleapis.com"), clusterWatcherCaptor1.capture()); - private FakeSubchannel(List eags) { - this.eags = eags; - } + UpstreamTlsContext upstreamTlsContext = + CommonTlsContextTestsUtil.buildUpstreamTlsContextFromFilenames( + CLIENT_KEY_FILE, CLIENT_PEM_FILE, CA_PEM_FILE); + + SslContextProvider mockSslContextProvider = mock(SslContextProvider.class); + doReturn(upstreamTlsContext).when(mockSslContextProvider).getUpstreamTlsContext(); + doReturn(mockSslContextProvider).when(mockTlsContextManager) + .findOrCreateClientSslContextProvider(same(upstreamTlsContext)); + + ClusterWatcher clusterWatcher1 = clusterWatcherCaptor1.getValue(); + clusterWatcher1.onClusterChanged( + ClusterUpdate.newBuilder() + .setClusterName("foo.googleapis.com") + .setEdsServiceName("edsServiceFoo.googleapis.com") + .setLbPolicy("round_robin") + .setUpstreamTlsContext(upstreamTlsContext) + .build()); - @Override - public void shutdown() { - } + assertThat(edsLbHelpers).hasSize(1); + assertThat(edsLoadBalancers).hasSize(1); + verify(mockTlsContextManager, never()) + .releaseClientSslContextProvider(any(SslContextProvider.class)); + Helper edsLbHelper1 = edsLbHelpers.poll(); + + ArrayList eagList = new ArrayList<>(); + eagList.add(new EquivalentAddressGroup(new InetSocketAddress("foo.com", 8080))); + eagList.add(new EquivalentAddressGroup(InetSocketAddress.createUnresolved("localhost", 8081), + Attributes.newBuilder().set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool).build())); + LoadBalancer.CreateSubchannelArgs createSubchannelArgs = + LoadBalancer.CreateSubchannelArgs.newBuilder() + .setAddresses(eagList) + .build(); + ArgumentCaptor createSubchannelArgsCaptor1 = + ArgumentCaptor.forClass(null); + verify(helper, never()) + .createSubchannel(any(LoadBalancer.CreateSubchannelArgs.class)); + edsLbHelper1.createSubchannel(createSubchannelArgs); + verifyUpstreamTlsContextAttribute(upstreamTlsContext, + createSubchannelArgsCaptor1); + + // update with same upstreamTlsContext + reset(mockTlsContextManager); + clusterWatcher1.onClusterChanged( + ClusterUpdate.newBuilder() + .setClusterName("bar.googleapis.com") + .setEdsServiceName("eds1ServiceFoo.googleapis.com") + .setLbPolicy("round_robin") + .setUpstreamTlsContext(upstreamTlsContext) + .build()); - @Override - public void requestConnection() { - } + verify(mockTlsContextManager, never()) + .releaseClientSslContextProvider(any(SslContextProvider.class)); + verify(mockTlsContextManager, never()).findOrCreateClientSslContextProvider( + any(UpstreamTlsContext.class)); - @Override - public List getAllAddresses() { - return eags; - } + // update with different upstreamTlsContext + reset(mockTlsContextManager); + reset(helper); + UpstreamTlsContext upstreamTlsContext1 = + CommonTlsContextTestsUtil.buildUpstreamTlsContextFromFilenames( + BAD_CLIENT_KEY_FILE, BAD_CLIENT_PEM_FILE, CA_PEM_FILE); + SslContextProvider mockSslContextProvider1 = mock(SslContextProvider.class); + doReturn(upstreamTlsContext1).when(mockSslContextProvider1).getUpstreamTlsContext(); + doReturn(mockSslContextProvider1).when(mockTlsContextManager) + .findOrCreateClientSslContextProvider(same(upstreamTlsContext1)); + clusterWatcher1.onClusterChanged( + ClusterUpdate.newBuilder() + .setClusterName("bar.googleapis.com") + .setEdsServiceName("eds1ServiceFoo.googleapis.com") + .setLbPolicy("round_robin") + .setUpstreamTlsContext(upstreamTlsContext1) + .build()); - @Override - public Attributes getAttributes() { - return Attributes.EMPTY; - } + verify(mockTlsContextManager).releaseClientSslContextProvider(same(mockSslContextProvider)); + verify(mockTlsContextManager).findOrCreateClientSslContextProvider(same(upstreamTlsContext1)); + ArgumentCaptor createSubchannelArgsCaptor2 = + ArgumentCaptor.forClass(null); + edsLbHelper1.createSubchannel(createSubchannelArgs); + verifyUpstreamTlsContextAttribute(upstreamTlsContext1, + createSubchannelArgsCaptor2); + + // update with null + reset(mockTlsContextManager); + reset(helper); + clusterWatcher1.onClusterChanged( + ClusterUpdate.newBuilder() + .setClusterName("bar.googleapis.com") + .setEdsServiceName("eds1ServiceFoo.googleapis.com") + .setLbPolicy("round_robin") + .setUpstreamTlsContext(null) + .build()); + verify(mockTlsContextManager).releaseClientSslContextProvider(same(mockSslContextProvider1)); + verify(mockTlsContextManager, never()).findOrCreateClientSslContextProvider( + any(UpstreamTlsContext.class)); + ArgumentCaptor createSubchannelArgsCaptor3 = + ArgumentCaptor.forClass(null); + edsLbHelper1.createSubchannel(createSubchannelArgs); + verifyUpstreamTlsContextAttribute(null, + createSubchannelArgsCaptor3); + + LoadBalancer edsLoadBalancer1 = edsLoadBalancers.poll(); + + cdsLoadBalancer.shutdown(); + verify(edsLoadBalancer1).shutdown(); + verify(xdsClient).cancelClusterDataWatch("foo.googleapis.com", clusterWatcher1); + assertThat(xdsClientPool.xdsClient).isNull(); } - private static final class FakeTlsContextManager implements TlsContextManager { + private void verifyUpstreamTlsContextAttribute( + UpstreamTlsContext upstreamTlsContext, + ArgumentCaptor createSubchannelArgsCaptor1) { + verify(helper, times(1)).createSubchannel(createSubchannelArgsCaptor1.capture()); + CreateSubchannelArgs capturedValue = createSubchannelArgsCaptor1.getValue(); + List capturedEagList = capturedValue.getAddresses(); + assertThat(capturedEagList.size()).isEqualTo(2); + EquivalentAddressGroup capturedEag = capturedEagList.get(0); + UpstreamTlsContext capturedUpstreamTlsContext = + capturedEag.getAttributes().get(XdsAttributes.ATTR_UPSTREAM_TLS_CONTEXT); + assertThat(capturedUpstreamTlsContext).isSameInstanceAs(upstreamTlsContext); + capturedEag = capturedEagList.get(1); + capturedUpstreamTlsContext = + capturedEag.getAttributes().get(XdsAttributes.ATTR_UPSTREAM_TLS_CONTEXT); + assertThat(capturedUpstreamTlsContext).isSameInstanceAs(upstreamTlsContext); + assertThat(capturedEag.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL)) + .isSameInstanceAs(xdsClientPool); + } - @Override - public SslContextProvider findOrCreateClientSslContextProvider( - UpstreamTlsContext upstreamTlsContext) { - SslContextProvider sslContextProvider = mock(SslContextProvider.class); - when(sslContextProvider.getUpstreamTlsContext()).thenReturn(upstreamTlsContext); - return sslContextProvider; - } + @Test + public void clusterWatcher_resourceNotExist() { + ResolvedAddresses resolvedAddresses = ResolvedAddresses.newBuilder() + .setAddresses(ImmutableList.of()) + .setAttributes(Attributes.newBuilder() + .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) + .build()) + .setLoadBalancingPolicyConfig(new CdsConfig("foo.googleapis.com")) + .build(); + cdsLoadBalancer.handleResolvedAddresses(resolvedAddresses); + + ArgumentCaptor clusterWatcherCaptor = ArgumentCaptor.forClass(null); + verify(xdsClient).watchClusterData(eq("foo.googleapis.com"), clusterWatcherCaptor.capture()); + + ClusterWatcher clusterWatcher = clusterWatcherCaptor.getValue(); + ArgumentCaptor pickerCaptor = ArgumentCaptor.forClass(null); + clusterWatcher.onResourceDoesNotExist("foo.googleapis.com"); + assertThat(edsLoadBalancers).isEmpty(); + verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); + PickResult result = pickerCaptor.getValue().pickSubchannel(mock(PickSubchannelArgs.class)); + assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAVAILABLE); + assertThat(result.getStatus().getDescription()) + .isEqualTo("Resource foo.googleapis.com is unavailable"); + } - @Override - public SslContextProvider releaseClientSslContextProvider( - SslContextProvider sslContextProvider) { - // no-op - return null; - } + @Test + public void clusterWatcher_resourceRemoved() { + ResolvedAddresses resolvedAddresses = ResolvedAddresses.newBuilder() + .setAddresses(ImmutableList.of()) + .setAttributes(Attributes.newBuilder() + .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) + .build()) + .setLoadBalancingPolicyConfig(new CdsConfig("foo.googleapis.com")) + .build(); + cdsLoadBalancer.handleResolvedAddresses(resolvedAddresses); + + ArgumentCaptor clusterWatcherCaptor = ArgumentCaptor.forClass(null); + verify(xdsClient).watchClusterData(eq("foo.googleapis.com"), clusterWatcherCaptor.capture()); + + ClusterWatcher clusterWatcher = clusterWatcherCaptor.getValue(); + ArgumentCaptor pickerCaptor = ArgumentCaptor.forClass(null); + clusterWatcher.onClusterChanged( + ClusterUpdate.newBuilder() + .setClusterName("foo.googleapis.com") + .setEdsServiceName("edsServiceFoo.googleapis.com") + .setLbPolicy("round_robin") + .build()); + assertThat(edsLoadBalancers).hasSize(1); + assertThat(edsLbHelpers).hasSize(1); + LoadBalancer edsLoadBalancer = edsLoadBalancers.poll(); + Helper edsHelper = edsLbHelpers.poll(); + SubchannelPicker subchannelPicker = mock(SubchannelPicker.class); + edsHelper.updateBalancingState(READY, subchannelPicker); + verify(helper).updateBalancingState(eq(READY), same(subchannelPicker)); + + clusterWatcher.onResourceDoesNotExist("foo.googleapis.com"); + verify(edsLoadBalancer).shutdown(); + verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); + PickResult result = pickerCaptor.getValue().pickSubchannel(mock(PickSubchannelArgs.class)); + assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAVAILABLE); + assertThat(result.getStatus().getDescription()) + .isEqualTo("Resource foo.googleapis.com is unavailable"); + } - @Override - public SslContextProvider findOrCreateServerSslContextProvider( - DownstreamTlsContext downstreamTlsContext) { - throw new UnsupportedOperationException("should not be called"); - } + @Test + public void clusterWatcher_onErrorCalledBeforeAndAfterOnClusterChanged() { + ResolvedAddresses resolvedAddresses = ResolvedAddresses.newBuilder() + .setAddresses(ImmutableList.of()) + .setAttributes(Attributes.newBuilder() + .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) + .build()) + .setLoadBalancingPolicyConfig(new CdsConfig("foo.googleapis.com")) + .build(); + cdsLoadBalancer.handleResolvedAddresses(resolvedAddresses); + + ArgumentCaptor clusterWatcherCaptor = ArgumentCaptor.forClass(null); + verify(xdsClient).watchClusterData(eq("foo.googleapis.com"), clusterWatcherCaptor.capture()); + + ClusterWatcher clusterWatcher = clusterWatcherCaptor.getValue(); + + // Call onError() before onClusterChanged() ever called. + clusterWatcher.onError(Status.DATA_LOSS.withDescription("fake status")); + assertThat(edsLoadBalancers).isEmpty(); + verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); + + clusterWatcher.onClusterChanged( + ClusterUpdate.newBuilder() + .setClusterName("foo.googleapis.com") + .setEdsServiceName("edsServiceFoo.googleapis.com") + .setLbPolicy("round_robin") + .build()); - @Override - public SslContextProvider releaseServerSslContextProvider( - SslContextProvider sslContextProvider) { - throw new UnsupportedOperationException("should not be called"); - } + assertThat(edsLbHelpers).hasSize(1); + assertThat(edsLoadBalancers).hasSize(1); + Helper edsLbHelper = edsLbHelpers.poll(); + LoadBalancer edsLoadBalancer = edsLoadBalancers.poll(); + verify(edsLoadBalancer).handleResolvedAddresses(any(ResolvedAddresses.class)); + SubchannelPicker picker = mock(SubchannelPicker.class); + + edsLbHelper.updateBalancingState(ConnectivityState.READY, picker); + verify(helper).updateBalancingState(ConnectivityState.READY, picker); + + // Call onError() after onClusterChanged(). + clusterWatcher.onError(Status.DATA_LOSS.withDescription("fake status")); + // Verify no more TRANSIENT_FAILURE. + verify(helper, times(1)) + .updateBalancingState(eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); } } From c7f876d0164edcd943aecbd211031821d0697536 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Tue, 1 Sep 2020 19:05:25 -0400 Subject: [PATCH 09/86] core: add default implementation of managed channel builder --- .../grpc/benchmarks/TransportBenchmark.java | 4 +- .../AbstractManagedChannelImplBuilder.java | 2 +- .../internal/ManagedChannelImplBuilder.java | 198 ++++++++++++++++++ .../ManagedChannelImplBuilderTest.java | 162 ++++++++++++++ .../ManagedChannelImplIdlenessTest.java | 23 +- .../grpc/internal/ManagedChannelImplTest.java | 67 +++--- .../ServiceConfigErrorHandlingTest.java | 43 ++-- 7 files changed, 418 insertions(+), 81 deletions(-) create mode 100644 core/src/main/java/io/grpc/internal/ManagedChannelImplBuilder.java create mode 100644 core/src/test/java/io/grpc/internal/ManagedChannelImplBuilderTest.java diff --git a/benchmarks/src/jmh/java/io/grpc/benchmarks/TransportBenchmark.java b/benchmarks/src/jmh/java/io/grpc/benchmarks/TransportBenchmark.java index 9f01b5e9143..ec4646bff44 100644 --- a/benchmarks/src/jmh/java/io/grpc/benchmarks/TransportBenchmark.java +++ b/benchmarks/src/jmh/java/io/grpc/benchmarks/TransportBenchmark.java @@ -20,6 +20,7 @@ import com.google.protobuf.ByteString; import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; import io.grpc.Server; import io.grpc.Status; import io.grpc.StatusRuntimeException; @@ -30,7 +31,6 @@ import io.grpc.benchmarks.qps.AsyncServer; import io.grpc.inprocess.InProcessChannelBuilder; import io.grpc.inprocess.InProcessServerBuilder; -import io.grpc.internal.AbstractManagedChannelImplBuilder; import io.grpc.internal.AbstractServerImplBuilder; import io.grpc.netty.NegotiationType; import io.grpc.netty.NettyChannelBuilder; @@ -81,7 +81,7 @@ public enum Transport { @Setup public void setUp() throws Exception { AbstractServerImplBuilder serverBuilder; - AbstractManagedChannelImplBuilder channelBuilder; + ManagedChannelBuilder channelBuilder; switch (transport) { case INPROCESS: { diff --git a/core/src/main/java/io/grpc/internal/AbstractManagedChannelImplBuilder.java b/core/src/main/java/io/grpc/internal/AbstractManagedChannelImplBuilder.java index b92fdf5410a..aac6c25a8ee 100644 --- a/core/src/main/java/io/grpc/internal/AbstractManagedChannelImplBuilder.java +++ b/core/src/main/java/io/grpc/internal/AbstractManagedChannelImplBuilder.java @@ -51,7 +51,7 @@ import javax.annotation.Nullable; /** - * The base class for channel builders. + * Abstract base class for channel builders. * * @param The concrete type of this builder. */ diff --git a/core/src/main/java/io/grpc/internal/ManagedChannelImplBuilder.java b/core/src/main/java/io/grpc/internal/ManagedChannelImplBuilder.java new file mode 100644 index 00000000000..02af7ef048c --- /dev/null +++ b/core/src/main/java/io/grpc/internal/ManagedChannelImplBuilder.java @@ -0,0 +1,198 @@ +/* + * Copyright 2020 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.internal; + +import com.google.common.base.Preconditions; +import io.grpc.ManagedChannelBuilder; +import java.net.SocketAddress; +import java.util.concurrent.Executor; +import javax.annotation.Nullable; + +/** + * Default managed channel builder, for usage in Transport implementations. + */ +public final class ManagedChannelImplBuilder + extends AbstractManagedChannelImplBuilder { + + private boolean authorityCheckerDisabled; + @Deprecated + @Nullable + private OverrideAuthorityChecker authorityChecker; + + /** + * An interface for Transport implementors to provide the {@link ClientTransportFactory} + * appropriate for the channel. + */ + public interface ClientTransportFactoryBuilder { + ClientTransportFactory buildClientTransportFactory(); + } + + /** + * An interface for Transport implementors to provide a default port to {@link + * io.grpc.NameResolver} for use in cases where the target string doesn't include a port. The + * default implementation returns {@link GrpcUtil#DEFAULT_PORT_SSL}. + */ + public interface ChannelBuilderDefaultPortProvider { + int getDefaultPort(); + } + + /** + * Default implementation of {@link ChannelBuilderDefaultPortProvider} that returns a fixed port. + */ + public static final class FixedPortProvider implements ChannelBuilderDefaultPortProvider { + private final int port; + + public FixedPortProvider(int port) { + this.port = port; + } + + @Override + public int getDefaultPort() { + return port; + } + } + + private final class ManagedChannelDefaultPortProvider implements + ChannelBuilderDefaultPortProvider { + @Override + public int getDefaultPort() { + return ManagedChannelImplBuilder.super.getDefaultPort(); + } + } + + private final ClientTransportFactoryBuilder clientTransportFactoryBuilder; + private final ChannelBuilderDefaultPortProvider channelBuilderDefaultPortProvider; + + /** + * Creates a new managed channel builder with a target string, which can be either a valid {@link + * io.grpc.NameResolver}-compliant URI, or an authority string. Transport implementors must + * provide client transport factory builder, and may set custom channel default port provider. + */ + public ManagedChannelImplBuilder(String target, + ClientTransportFactoryBuilder clientTransportFactoryBuilder, + @Nullable ChannelBuilderDefaultPortProvider channelBuilderDefaultPortProvider) { + super(target); + this.clientTransportFactoryBuilder = Preconditions + .checkNotNull(clientTransportFactoryBuilder, "clientTransportFactoryBuilder"); + + if (channelBuilderDefaultPortProvider != null) { + this.channelBuilderDefaultPortProvider = channelBuilderDefaultPortProvider; + } else { + this.channelBuilderDefaultPortProvider = new ManagedChannelDefaultPortProvider(); + } + } + + /** + * Creates a new managed channel builder with the given server address, authority string of the + * channel. Transport implementors must provide client transport factory builder, and may set + * custom channel default port provider. + */ + public ManagedChannelImplBuilder(SocketAddress directServerAddress, String authority, + ClientTransportFactoryBuilder clientTransportFactoryBuilder, + @Nullable ChannelBuilderDefaultPortProvider channelBuilderDefaultPortProvider) { + super(directServerAddress, authority); + this.clientTransportFactoryBuilder = Preconditions + .checkNotNull(clientTransportFactoryBuilder, "clientTransportFactoryBuilder"); + + if (channelBuilderDefaultPortProvider != null) { + this.channelBuilderDefaultPortProvider = channelBuilderDefaultPortProvider; + } else { + this.channelBuilderDefaultPortProvider = new ManagedChannelDefaultPortProvider(); + } + } + + @Override + protected ClientTransportFactory buildTransportFactory() { + return clientTransportFactoryBuilder.buildClientTransportFactory(); + } + + @Override + protected int getDefaultPort() { + return channelBuilderDefaultPortProvider.getDefaultPort(); + } + + /** Disable the check whether the authority is valid. */ + public ManagedChannelImplBuilder disableCheckAuthority() { + authorityCheckerDisabled = true; + return this; + } + + /** Enable previously disabled authority check. */ + public ManagedChannelImplBuilder enableCheckAuthority() { + authorityCheckerDisabled = false; + return this; + } + + @Deprecated + public interface OverrideAuthorityChecker { + String checkAuthority(String authority); + } + + @Deprecated + public void overrideAuthorityChecker(@Nullable OverrideAuthorityChecker authorityChecker) { + this.authorityChecker = authorityChecker; + } + + @Override + protected String checkAuthority(String authority) { + if (authorityCheckerDisabled) { + return authority; + } + if (authorityChecker != null) { + return authorityChecker.checkAuthority(authority); + } + return super.checkAuthority(authority); + } + + @Override + public void setStatsEnabled(boolean value) { + super.setStatsEnabled(value); + } + + @Override + public void setStatsRecordStartedRpcs(boolean value) { + super.setStatsRecordStartedRpcs(value); + } + + @Override + public void setStatsRecordFinishedRpcs(boolean value) { + super.setStatsRecordFinishedRpcs(value); + } + + @Override + public void setStatsRecordRealTimeMetrics(boolean value) { + super.setStatsRecordRealTimeMetrics(value); + } + + @Override + public void setTracingEnabled(boolean value) { + super.setTracingEnabled(value); + } + + @Override + public ObjectPool getOffloadExecutorPool() { + return super.getOffloadExecutorPool(); + } + + public static ManagedChannelBuilder forAddress(String name, int port) { + throw new UnsupportedOperationException("ClientTransportFactoryBuilder is required"); + } + + public static ManagedChannelBuilder forTarget(String target) { + throw new UnsupportedOperationException("ClientTransportFactoryBuilder is required"); + } +} diff --git a/core/src/test/java/io/grpc/internal/ManagedChannelImplBuilderTest.java b/core/src/test/java/io/grpc/internal/ManagedChannelImplBuilderTest.java new file mode 100644 index 00000000000..19a2f800965 --- /dev/null +++ b/core/src/test/java/io/grpc/internal/ManagedChannelImplBuilderTest.java @@ -0,0 +1,162 @@ +/* + * Copyright 2020 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.internal; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import io.grpc.internal.ManagedChannelImplBuilder.ChannelBuilderDefaultPortProvider; +import io.grpc.internal.ManagedChannelImplBuilder.ClientTransportFactoryBuilder; +import io.grpc.internal.ManagedChannelImplBuilder.FixedPortProvider; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** Unit tests for {@link ManagedChannelImplBuilder}. */ +@RunWith(JUnit4.class) +public class ManagedChannelImplBuilderTest { + private static final int DUMMY_PORT = 42; + private static final String DUMMY_TARGET = "fake-target"; + private static final String DUMMY_AUTHORITY_VALID = "valid:1234"; + private static final String DUMMY_AUTHORITY_INVALID = "[ : : 1]"; + + @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + @Rule public final ExpectedException thrown = ExpectedException.none(); + + @Mock private ClientTransportFactoryBuilder mockClientTransportFactoryBuilder; + @Mock private ChannelBuilderDefaultPortProvider mockChannelBuilderDefaultPortProvider; + private ManagedChannelImplBuilder builder; + + @Before + public void setUp() throws Exception { + builder = new ManagedChannelImplBuilder( + DUMMY_TARGET, + mockClientTransportFactoryBuilder, + mockChannelBuilderDefaultPortProvider); + } + + /** Ensure buildTransportFactory() delegates to the custom implementation. */ + @Test + public void buildTransportFactory() { + final ClientTransportFactory clientTransportFactory = mock(ClientTransportFactory.class); + when(mockClientTransportFactoryBuilder.buildClientTransportFactory()) + .thenReturn(clientTransportFactory); + assertEquals(clientTransportFactory, builder.buildTransportFactory()); + verify(mockClientTransportFactoryBuilder).buildClientTransportFactory(); + } + + /** Ensure getDefaultPort() returns default port when no custom implementation provided. */ + @Test + public void getDefaultPort_default() { + final ManagedChannelImplBuilder builderNoPortProvider = new ManagedChannelImplBuilder( + DUMMY_TARGET, mockClientTransportFactoryBuilder, null); + assertEquals(GrpcUtil.DEFAULT_PORT_SSL, builderNoPortProvider.getDefaultPort()); + } + + /** Ensure getDefaultPort() delegates to the custom implementation. */ + @Test + public void getDefaultPort_custom() { + when(mockChannelBuilderDefaultPortProvider.getDefaultPort()).thenReturn(DUMMY_PORT); + assertEquals(DUMMY_PORT, builder.getDefaultPort()); + verify(mockChannelBuilderDefaultPortProvider).getDefaultPort(); + } + + /** Test FixedPortProvider(int port). */ + @Test + public void getDefaultPort_fixedPortProvider() { + final ManagedChannelImplBuilder builderFixedPortProvider = new ManagedChannelImplBuilder( + DUMMY_TARGET, + mockClientTransportFactoryBuilder, + new FixedPortProvider(DUMMY_PORT)); + assertEquals(DUMMY_PORT, builderFixedPortProvider.getDefaultPort()); + } + + @Test + public void checkAuthority_validAuthorityAllowed() { + assertEquals(DUMMY_AUTHORITY_VALID, builder.checkAuthority(DUMMY_AUTHORITY_VALID)); + } + + @Test + public void checkAuthority_invalidAuthorityFailed() { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("Invalid authority"); + + builder.checkAuthority(DUMMY_AUTHORITY_INVALID); + } + + @Test + public void disableCheckAuthority_validAuthorityAllowed() { + builder.disableCheckAuthority(); + assertEquals(DUMMY_AUTHORITY_VALID, builder.checkAuthority(DUMMY_AUTHORITY_VALID)); + } + + @Test + public void disableCheckAuthority_invalidAuthorityAllowed() { + builder.disableCheckAuthority(); + assertEquals(DUMMY_AUTHORITY_INVALID, builder.checkAuthority(DUMMY_AUTHORITY_INVALID)); + } + + @Test + public void enableCheckAuthority_validAuthorityAllowed() { + builder.disableCheckAuthority().enableCheckAuthority(); + assertEquals(DUMMY_AUTHORITY_VALID, builder.checkAuthority(DUMMY_AUTHORITY_VALID)); + } + + @Test + public void disableCheckAuthority_invalidAuthorityFailed() { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("Invalid authority"); + + builder.disableCheckAuthority().enableCheckAuthority(); + builder.checkAuthority(DUMMY_AUTHORITY_INVALID); + } + + /** Ensure authority check can disabled with custom authority check implementation. */ + @Test + @SuppressWarnings("deprecation") + public void overrideAuthorityChecker_default() { + builder.overrideAuthorityChecker( + new io.grpc.internal.ManagedChannelImplBuilder.OverrideAuthorityChecker() { + @Override public String checkAuthority(String authority) { + return authority; + } + }); + assertEquals(DUMMY_AUTHORITY_INVALID, builder.checkAuthority(DUMMY_AUTHORITY_INVALID)); + } + + /** Ensure custom authority is ignored after disableCheckAuthority(). */ + @Test + @SuppressWarnings("deprecation") + public void overrideAuthorityChecker_ignored() { + builder.overrideAuthorityChecker( + new io.grpc.internal.ManagedChannelImplBuilder.OverrideAuthorityChecker() { + @Override public String checkAuthority(String authority) { + throw new IllegalArgumentException(); + } + }); + builder.disableCheckAuthority(); + assertEquals(DUMMY_AUTHORITY_INVALID, builder.checkAuthority(DUMMY_AUTHORITY_INVALID)); + } +} diff --git a/core/src/test/java/io/grpc/internal/ManagedChannelImplIdlenessTest.java b/core/src/test/java/io/grpc/internal/ManagedChannelImplIdlenessTest.java index 2295dcb0ff2..c551d26449c 100644 --- a/core/src/test/java/io/grpc/internal/ManagedChannelImplIdlenessTest.java +++ b/core/src/test/java/io/grpc/internal/ManagedChannelImplIdlenessTest.java @@ -62,6 +62,7 @@ import io.grpc.Status; import io.grpc.StringMarshaller; import io.grpc.internal.FakeClock.ScheduledTask; +import io.grpc.internal.ManagedChannelImplBuilder.ClientTransportFactoryBuilder; import io.grpc.internal.TestUtils.MockClientTransportInfo; import java.net.SocketAddress; import java.net.URI; @@ -159,21 +160,15 @@ public void setUp() { when(mockTransportFactory.getScheduledExecutorService()) .thenReturn(timer.getScheduledExecutorService()); - class Builder extends AbstractManagedChannelImplBuilder { - Builder(String target) { - super(target); - } - - @Override protected ClientTransportFactory buildTransportFactory() { - throw new UnsupportedOperationException(); - } - - @Override public Builder usePlaintext() { - throw new UnsupportedOperationException(); - } - } + ManagedChannelImplBuilder builder = new ManagedChannelImplBuilder("fake://target", + new ClientTransportFactoryBuilder() { + @Override public ClientTransportFactory buildClientTransportFactory() { + throw new UnsupportedOperationException(); + } + }, + null); - Builder builder = new Builder("fake://target") + builder .nameResolverFactory(mockNameResolverFactory) .defaultLoadBalancingPolicy(MOCK_POLICY_NAME) .idleTimeout(IDLE_TIMEOUT_SECONDS, TimeUnit.SECONDS) diff --git a/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java b/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java index 3650911f142..ba17deba613 100644 --- a/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java +++ b/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java @@ -107,6 +107,8 @@ import io.grpc.internal.ClientTransportFactory.ClientTransportOptions; import io.grpc.internal.InternalSubchannel.TransportLogger; import io.grpc.internal.ManagedChannelImpl.ScParser; +import io.grpc.internal.ManagedChannelImplBuilder.ClientTransportFactoryBuilder; +import io.grpc.internal.ManagedChannelImplBuilder.FixedPortProvider; import io.grpc.internal.ServiceConfigUtil.PolicySelection; import io.grpc.internal.TestUtils.MockClientTransportInfo; import io.grpc.stub.ClientCalls; @@ -269,7 +271,7 @@ public String getPolicyName() { private CallCredentials creds; @Mock private Executor offloadExecutor; - private ChannelBuilder channelBuilder; + private ManagedChannelImplBuilder channelBuilder; private boolean requestConnection = true; private BlockingQueue transports; private boolean panicExpected; @@ -325,14 +327,20 @@ public void setUp() throws Exception { when(balancerRpcExecutorPool.getObject()) .thenReturn(balancerRpcExecutor.getScheduledExecutorService()); - channelBuilder = - new ChannelBuilder() - .nameResolverFactory(new FakeNameResolverFactory.Builder(expectedUri).build()) - .defaultLoadBalancingPolicy(MOCK_POLICY_NAME) - .userAgent(USER_AGENT) - .idleTimeout( - AbstractManagedChannelImplBuilder.IDLE_MODE_MAX_TIMEOUT_DAYS, TimeUnit.DAYS) - .offloadExecutor(offloadExecutor); + channelBuilder = new ManagedChannelImplBuilder(TARGET, + new ClientTransportFactoryBuilder() { + @Override + public ClientTransportFactory buildClientTransportFactory() { + throw new UnsupportedOperationException(); + } + }, + new FixedPortProvider(DEFAULT_PORT)); + channelBuilder + .nameResolverFactory(new FakeNameResolverFactory.Builder(expectedUri).build()) + .defaultLoadBalancingPolicy(MOCK_POLICY_NAME) + .userAgent(USER_AGENT) + .idleTimeout(ManagedChannelImplBuilder.IDLE_MODE_MAX_TIMEOUT_DAYS, TimeUnit.DAYS) + .offloadExecutor(offloadExecutor); channelBuilder.executorPool = executorPool; channelBuilder.binlog = null; channelBuilder.channelz = channelz; @@ -3466,21 +3474,18 @@ public String getDefaultScheme() { } FakeNameResolverFactory2 factory = new FakeNameResolverFactory2(); - final class CustomBuilder extends AbstractManagedChannelImplBuilder { - - CustomBuilder() { - super(TARGET); - this.executorPool = ManagedChannelImplTest.this.executorPool; - this.channelz = ManagedChannelImplTest.this.channelz; - } - @Override - protected ClientTransportFactory buildTransportFactory() { - return mockTransportFactory; - } - } - - ManagedChannel mychannel = new CustomBuilder().nameResolverFactory(factory).build(); + ManagedChannelImplBuilder customBuilder = new ManagedChannelImplBuilder(TARGET, + new ClientTransportFactoryBuilder() { + @Override + public ClientTransportFactory buildClientTransportFactory() { + return mockTransportFactory; + } + }, + null); + customBuilder.executorPool = executorPool; + customBuilder.channelz = channelz; + ManagedChannel mychannel = customBuilder.nameResolverFactory(factory).build(); ClientCall call1 = mychannel.newCall(TestMethodDescriptors.voidMethod(), CallOptions.DEFAULT); @@ -4025,22 +4030,6 @@ public void createResolvingOobChannel() throws Exception { } } - private static final class ChannelBuilder - extends AbstractManagedChannelImplBuilder { - - ChannelBuilder() { - super(TARGET); - } - - @Override protected ClientTransportFactory buildTransportFactory() { - throw new UnsupportedOperationException(); - } - - @Override protected int getDefaultPort() { - return DEFAULT_PORT; - } - } - private static final class FakeBackoffPolicyProvider implements BackoffPolicy.Provider { @Override public BackoffPolicy get() { diff --git a/core/src/test/java/io/grpc/internal/ServiceConfigErrorHandlingTest.java b/core/src/test/java/io/grpc/internal/ServiceConfigErrorHandlingTest.java index de6098e5bd7..49f094a6adb 100644 --- a/core/src/test/java/io/grpc/internal/ServiceConfigErrorHandlingTest.java +++ b/core/src/test/java/io/grpc/internal/ServiceConfigErrorHandlingTest.java @@ -46,6 +46,8 @@ import io.grpc.NameResolver; import io.grpc.NameResolver.ConfigOrError; import io.grpc.Status; +import io.grpc.internal.ManagedChannelImplBuilder.ClientTransportFactoryBuilder; +import io.grpc.internal.ManagedChannelImplBuilder.FixedPortProvider; import java.net.SocketAddress; import java.net.URI; import java.util.ArrayList; @@ -151,7 +153,7 @@ public ConfigOrError parseLoadBalancingPolicyConfig( private ObjectPool balancerRpcExecutorPool; @Mock private Executor blockingExecutor; - private ChannelBuilder channelBuilder; + private ManagedChannelImplBuilder channelBuilder; private void createChannel(ClientInterceptor... interceptors) { checkState(channel == null); @@ -197,14 +199,21 @@ public void setUp() throws Exception { .thenReturn(timer.getScheduledExecutorService()); when(executorPool.getObject()).thenReturn(executor.getScheduledExecutorService()); - channelBuilder = - new ChannelBuilder() - .nameResolverFactory(new FakeNameResolverFactory.Builder(expectedUri).build()) - .defaultLoadBalancingPolicy(MOCK_POLICY_NAME) - .userAgent(USER_AGENT) - .idleTimeout( - AbstractManagedChannelImplBuilder.IDLE_MODE_MAX_TIMEOUT_DAYS, TimeUnit.DAYS) - .offloadExecutor(blockingExecutor); + channelBuilder = new ManagedChannelImplBuilder(TARGET, + new ClientTransportFactoryBuilder() { + @Override + public ClientTransportFactory buildClientTransportFactory() { + throw new UnsupportedOperationException(); + } + }, + new FixedPortProvider(DEFAULT_PORT)); + + channelBuilder + .nameResolverFactory(new FakeNameResolverFactory.Builder(expectedUri).build()) + .defaultLoadBalancingPolicy(MOCK_POLICY_NAME) + .userAgent(USER_AGENT) + .idleTimeout(ManagedChannelImplBuilder.IDLE_MODE_MAX_TIMEOUT_DAYS, TimeUnit.DAYS) + .offloadExecutor(blockingExecutor); channelBuilder.executorPool = executorPool; channelBuilder.binlog = null; channelBuilder.channelz = channelz; @@ -527,22 +536,6 @@ public void validConfig_thenNoConfig_withDefaultConfig() throws Exception { verify(mockLoadBalancer, never()).handleNameResolutionError(any(Status.class)); } - private static final class ChannelBuilder - extends AbstractManagedChannelImplBuilder { - - ChannelBuilder() { - super(TARGET); - } - - @Override protected ClientTransportFactory buildTransportFactory() { - throw new UnsupportedOperationException(); - } - - @Override protected int getDefaultPort() { - return DEFAULT_PORT; - } - } - private static final class FakeBackoffPolicyProvider implements BackoffPolicy.Provider { @Override public BackoffPolicy get() { From a429b9767cc004a52a6ad491e397b2290a5a979d Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Tue, 1 Sep 2020 19:33:45 -0400 Subject: [PATCH 10/86] cronet: CronetChannelBuilder extends a public API class --- .../io/grpc/cronet/CronetChannelBuilder.java | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/cronet/src/main/java/io/grpc/cronet/CronetChannelBuilder.java b/cronet/src/main/java/io/grpc/cronet/CronetChannelBuilder.java index 1a87de58507..3ed71e234aa 100644 --- a/cronet/src/main/java/io/grpc/cronet/CronetChannelBuilder.java +++ b/cronet/src/main/java/io/grpc/cronet/CronetChannelBuilder.java @@ -26,10 +26,14 @@ import com.google.common.util.concurrent.MoreExecutors; import io.grpc.ChannelLogger; import io.grpc.ExperimentalApi; -import io.grpc.internal.AbstractManagedChannelImplBuilder; +import io.grpc.ForwardingChannelBuilder; +import io.grpc.Internal; +import io.grpc.ManagedChannelBuilder; import io.grpc.internal.ClientTransportFactory; import io.grpc.internal.ConnectionClientTransport; import io.grpc.internal.GrpcUtil; +import io.grpc.internal.ManagedChannelImplBuilder; +import io.grpc.internal.ManagedChannelImplBuilder.ClientTransportFactoryBuilder; import io.grpc.internal.SharedResourceHolder; import io.grpc.internal.TransportTracer; import java.lang.reflect.InvocationTargetException; @@ -46,8 +50,7 @@ /** Convenience class for building channels with the cronet transport. */ @ExperimentalApi("There is no plan to make this API stable, given transport API instability") -public final class CronetChannelBuilder extends - AbstractManagedChannelImplBuilder { +public final class CronetChannelBuilder extends ForwardingChannelBuilder { private static final String LOG_TAG = "CronetChannelBuilder"; @@ -81,6 +84,8 @@ public static CronetChannelBuilder forAddress(String name, int port) { private ScheduledExecutorService scheduledExecutorService; private final CronetEngine cronetEngine; + private final ManagedChannelImplBuilder managedChannelImplBuilder; + private TransportTracer.Factory transportTracerFactory = TransportTracer.getDefaultFactory(); private boolean alwaysUsePut = false; @@ -103,12 +108,27 @@ public static CronetChannelBuilder forAddress(String name, int port) { private int trafficStatsUid; private CronetChannelBuilder(String host, int port, CronetEngine cronetEngine) { - super( + final class CronetChannelTransportFactoryBuilder implements ClientTransportFactoryBuilder { + @Override + public ClientTransportFactory buildClientTransportFactory() { + return buildTransportFactory(); + } + } + + managedChannelImplBuilder = new ManagedChannelImplBuilder( InetSocketAddress.createUnresolved(host, port), - GrpcUtil.authorityFromHostAndPort(host, port)); + GrpcUtil.authorityFromHostAndPort(host, port), + new CronetChannelTransportFactoryBuilder(), + null); this.cronetEngine = Preconditions.checkNotNull(cronetEngine, "cronetEngine"); } + @Internal + @Override + protected ManagedChannelBuilder delegate() { + return managedChannelImplBuilder; + } + /** * Sets the maximum message size allowed to be received on the channel. If not called, * defaults to {@link io.grpc.internal.GrpcUtil#DEFAULT_MAX_MESSAGE_SIZE}. @@ -188,8 +208,7 @@ public final CronetChannelBuilder scheduledExecutorService( return this; } - @Override - protected final ClientTransportFactory buildTransportFactory() { + ClientTransportFactory buildTransportFactory() { return new CronetTransportFactory( new TaggingStreamFactory( cronetEngine, trafficStatsTagSet, trafficStatsTag, trafficStatsUidSet, trafficStatsUid), From b0f0ed080ef2731e3124689d338ee9f8e74e3e87 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Tue, 1 Sep 2020 19:41:10 -0400 Subject: [PATCH 11/86] core: InProcessChannelBuilder extends a public API class --- .../inprocess/InProcessChannelBuilder.java | 38 +++++++++++++++---- .../InternalInProcessChannelBuilder.java | 33 ++++++++++++++++ .../testing/integration/InProcessTest.java | 3 +- 3 files changed, 65 insertions(+), 9 deletions(-) create mode 100644 core/src/main/java/io/grpc/inprocess/InternalInProcessChannelBuilder.java diff --git a/core/src/main/java/io/grpc/inprocess/InProcessChannelBuilder.java b/core/src/main/java/io/grpc/inprocess/InProcessChannelBuilder.java index 3546e33357c..926c7835729 100644 --- a/core/src/main/java/io/grpc/inprocess/InProcessChannelBuilder.java +++ b/core/src/main/java/io/grpc/inprocess/InProcessChannelBuilder.java @@ -21,11 +21,14 @@ import io.grpc.ChannelLogger; import io.grpc.ExperimentalApi; +import io.grpc.ForwardingChannelBuilder; import io.grpc.Internal; -import io.grpc.internal.AbstractManagedChannelImplBuilder; +import io.grpc.ManagedChannelBuilder; import io.grpc.internal.ClientTransportFactory; import io.grpc.internal.ConnectionClientTransport; import io.grpc.internal.GrpcUtil; +import io.grpc.internal.ManagedChannelImplBuilder; +import io.grpc.internal.ManagedChannelImplBuilder.ClientTransportFactoryBuilder; import io.grpc.internal.SharedResourceHolder; import java.net.SocketAddress; import java.util.concurrent.ScheduledExecutorService; @@ -42,7 +45,7 @@ */ @ExperimentalApi("https://github.com/grpc/grpc-java/issues/1783") public final class InProcessChannelBuilder extends - AbstractManagedChannelImplBuilder { + ForwardingChannelBuilder { /** * Create a channel builder that will connect to the server with the given name. * @@ -67,18 +70,35 @@ public static InProcessChannelBuilder forAddress(String name, int port) { throw new UnsupportedOperationException("call forName() instead"); } + private final ManagedChannelImplBuilder managedChannelImplBuilder; private final String name; private ScheduledExecutorService scheduledExecutorService; private int maxInboundMetadataSize = Integer.MAX_VALUE; private boolean transportIncludeStatusCause = false; private InProcessChannelBuilder(String name) { - super(new InProcessSocketAddress(name), "localhost"); this.name = checkNotNull(name, "name"); + + final class InProcessChannelTransportFactoryBuilder implements ClientTransportFactoryBuilder { + @Override + public ClientTransportFactory buildClientTransportFactory() { + return buildTransportFactory(); + } + } + + managedChannelImplBuilder = new ManagedChannelImplBuilder(new InProcessSocketAddress(name), + "localhost", new InProcessChannelTransportFactoryBuilder(), null); + // In-process transport should not record its traffic to the stats module. // https://github.com/grpc/grpc-java/issues/2284 - setStatsRecordStartedRpcs(false); - setStatsRecordFinishedRpcs(false); + managedChannelImplBuilder.setStatsRecordStartedRpcs(false); + managedChannelImplBuilder.setStatsRecordFinishedRpcs(false); + } + + @Internal + @Override + protected ManagedChannelBuilder delegate() { + return managedChannelImplBuilder; } @Override @@ -177,13 +197,15 @@ public InProcessChannelBuilder propagateCauseWithStatus(boolean enable) { return this; } - @Override - @Internal - protected ClientTransportFactory buildTransportFactory() { + ClientTransportFactory buildTransportFactory() { return new InProcessClientTransportFactory( name, scheduledExecutorService, maxInboundMetadataSize, transportIncludeStatusCause); } + void setStatsEnabled(boolean value) { + this.managedChannelImplBuilder.setStatsEnabled(value); + } + /** * Creates InProcess transports. Exposed for internal use, as it should be private. */ diff --git a/core/src/main/java/io/grpc/inprocess/InternalInProcessChannelBuilder.java b/core/src/main/java/io/grpc/inprocess/InternalInProcessChannelBuilder.java new file mode 100644 index 00000000000..1a017fe564c --- /dev/null +++ b/core/src/main/java/io/grpc/inprocess/InternalInProcessChannelBuilder.java @@ -0,0 +1,33 @@ +/* + * Copyright 2020 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.inprocess; + +import io.grpc.Internal; + +/** + * Internal {@link InProcessChannelBuilder} accessor. This is intended for usage internal to the + * gRPC team. If you *really* think you need to use this, contact the gRPC team first. + */ +@Internal +public final class InternalInProcessChannelBuilder { + + public static void setStatsEnabled(InProcessChannelBuilder builder, boolean value) { + builder.setStatsEnabled(value); + } + + private InternalInProcessChannelBuilder() {} +} diff --git a/interop-testing/src/test/java/io/grpc/testing/integration/InProcessTest.java b/interop-testing/src/test/java/io/grpc/testing/integration/InProcessTest.java index fd403affcf5..66894834b95 100644 --- a/interop-testing/src/test/java/io/grpc/testing/integration/InProcessTest.java +++ b/interop-testing/src/test/java/io/grpc/testing/integration/InProcessTest.java @@ -18,6 +18,7 @@ import io.grpc.inprocess.InProcessChannelBuilder; import io.grpc.inprocess.InProcessServerBuilder; +import io.grpc.inprocess.InternalInProcessChannelBuilder; import io.grpc.internal.AbstractServerImplBuilder; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -38,7 +39,7 @@ protected AbstractServerImplBuilder getServerBuilder() { protected InProcessChannelBuilder createChannelBuilder() { InProcessChannelBuilder builder = InProcessChannelBuilder.forName(SERVER_NAME); // Disable the default census stats interceptor, use testing interceptor instead. - io.grpc.internal.TestingAccessor.setStatsEnabled(builder, false); + InternalInProcessChannelBuilder.setStatsEnabled(builder, false); return builder.intercept(createCensusStatsClientInterceptor()); } From c0569796711ce93dff852217e76053e2296f3f0e Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Wed, 2 Sep 2020 11:18:10 -0400 Subject: [PATCH 12/86] netty: NettyChannelBuilder extends a public API class --- .../integration/TestServiceClient.java | 5 +- .../integration/AutoWindowSizingOnTest.java | 3 +- .../Http2NettyLocalChannelTest.java | 3 +- .../testing/integration/Http2NettyTest.java | 3 +- .../integration/TransportCompressionTest.java | 3 +- .../netty/InternalNettyChannelBuilder.java | 21 +++ .../io/grpc/netty/NettyChannelBuilder.java | 120 ++++++++++++------ .../grpc/netty/NettyChannelBuilderTest.java | 58 +++++++-- 8 files changed, 161 insertions(+), 55 deletions(-) diff --git a/interop-testing/src/main/java/io/grpc/testing/integration/TestServiceClient.java b/interop-testing/src/main/java/io/grpc/testing/integration/TestServiceClient.java index fa7010d19a8..c06a525c85b 100644 --- a/interop-testing/src/main/java/io/grpc/testing/integration/TestServiceClient.java +++ b/interop-testing/src/main/java/io/grpc/testing/integration/TestServiceClient.java @@ -27,6 +27,7 @@ import io.grpc.internal.GrpcUtil; import io.grpc.internal.testing.TestUtils; import io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.InternalNettyChannelBuilder; import io.grpc.netty.NegotiationType; import io.grpc.netty.NettyChannelBuilder; import io.grpc.okhttp.OkHttpChannelBuilder; @@ -425,7 +426,9 @@ protected ManagedChannelBuilder createChannelBuilder() { if (fullStreamDecompression) { nettyBuilder.enableFullStreamDecompression(); } - builder = nettyBuilder; + // Disable the default census stats interceptor, use testing interceptor instead. + InternalNettyChannelBuilder.setStatsEnabled(nettyBuilder, false); + return nettyBuilder.intercept(createCensusStatsClientInterceptor()); } else { OkHttpChannelBuilder okBuilder = OkHttpChannelBuilder.forAddress(serverHost, serverPort); if (serverHostOverride != null) { diff --git a/interop-testing/src/test/java/io/grpc/testing/integration/AutoWindowSizingOnTest.java b/interop-testing/src/test/java/io/grpc/testing/integration/AutoWindowSizingOnTest.java index a68181be831..a2036ecea91 100644 --- a/interop-testing/src/test/java/io/grpc/testing/integration/AutoWindowSizingOnTest.java +++ b/interop-testing/src/test/java/io/grpc/testing/integration/AutoWindowSizingOnTest.java @@ -17,6 +17,7 @@ package io.grpc.testing.integration; import io.grpc.internal.AbstractServerImplBuilder; +import io.grpc.netty.InternalNettyChannelBuilder; import io.grpc.netty.NegotiationType; import io.grpc.netty.NettyChannelBuilder; import io.grpc.netty.NettyServerBuilder; @@ -39,7 +40,7 @@ protected NettyChannelBuilder createChannelBuilder() { .maxInboundMessageSize(AbstractInteropTest.MAX_MESSAGE_SIZE) .initialFlowControlWindow(NettyChannelBuilder.DEFAULT_FLOW_CONTROL_WINDOW); // Disable the default census stats interceptor, use testing interceptor instead. - io.grpc.internal.TestingAccessor.setStatsEnabled(builder, false); + InternalNettyChannelBuilder.setStatsEnabled(builder, false); return builder.intercept(createCensusStatsClientInterceptor()); } } diff --git a/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyLocalChannelTest.java b/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyLocalChannelTest.java index 5592801755b..8ed7dc76900 100644 --- a/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyLocalChannelTest.java +++ b/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyLocalChannelTest.java @@ -17,6 +17,7 @@ package io.grpc.testing.integration; import io.grpc.internal.AbstractServerImplBuilder; +import io.grpc.netty.InternalNettyChannelBuilder; import io.grpc.netty.NegotiationType; import io.grpc.netty.NettyChannelBuilder; import io.grpc.netty.NettyServerBuilder; @@ -57,7 +58,7 @@ protected NettyChannelBuilder createChannelBuilder() { .flowControlWindow(65 * 1024) .maxInboundMessageSize(AbstractInteropTest.MAX_MESSAGE_SIZE); // Disable the default census stats interceptor, use testing interceptor instead. - io.grpc.internal.TestingAccessor.setStatsEnabled(builder, false); + InternalNettyChannelBuilder.setStatsEnabled(builder, false); return builder.intercept(createCensusStatsClientInterceptor()); } diff --git a/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyTest.java b/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyTest.java index 1819b9f3008..353180cbafb 100644 --- a/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyTest.java +++ b/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyTest.java @@ -22,6 +22,7 @@ import io.grpc.internal.AbstractServerImplBuilder; import io.grpc.internal.testing.TestUtils; import io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.InternalNettyChannelBuilder; import io.grpc.netty.NettyChannelBuilder; import io.grpc.netty.NettyServerBuilder; import io.netty.handler.ssl.ClientAuth; @@ -71,7 +72,7 @@ protected NettyChannelBuilder createChannelBuilder() { .ciphers(TestUtils.preferredTestCiphers(), SupportedCipherSuiteFilter.INSTANCE) .build()); // Disable the default census stats interceptor, use testing interceptor instead. - io.grpc.internal.TestingAccessor.setStatsEnabled(builder, false); + InternalNettyChannelBuilder.setStatsEnabled(builder, false); return builder.intercept(createCensusStatsClientInterceptor()); } catch (Exception ex) { throw new RuntimeException(ex); diff --git a/interop-testing/src/test/java/io/grpc/testing/integration/TransportCompressionTest.java b/interop-testing/src/test/java/io/grpc/testing/integration/TransportCompressionTest.java index 8bc0494b79d..1144c75073c 100644 --- a/interop-testing/src/test/java/io/grpc/testing/integration/TransportCompressionTest.java +++ b/interop-testing/src/test/java/io/grpc/testing/integration/TransportCompressionTest.java @@ -37,6 +37,7 @@ import io.grpc.ServerInterceptor; import io.grpc.internal.AbstractServerImplBuilder; import io.grpc.internal.GrpcUtil; +import io.grpc.netty.InternalNettyChannelBuilder; import io.grpc.netty.NettyChannelBuilder; import io.grpc.netty.NettyServerBuilder; import io.grpc.testing.integration.Messages.BoolValue; @@ -165,7 +166,7 @@ public void onHeaders(Metadata headers) { }) .usePlaintext(); // Disable the default census stats interceptor, use testing interceptor instead. - io.grpc.internal.TestingAccessor.setStatsEnabled(builder, false); + InternalNettyChannelBuilder.setStatsEnabled(builder, false); return builder.intercept(createCensusStatsClientInterceptor()); } diff --git a/netty/src/main/java/io/grpc/netty/InternalNettyChannelBuilder.java b/netty/src/main/java/io/grpc/netty/InternalNettyChannelBuilder.java index e66f28cac73..29423389193 100644 --- a/netty/src/main/java/io/grpc/netty/InternalNettyChannelBuilder.java +++ b/netty/src/main/java/io/grpc/netty/InternalNettyChannelBuilder.java @@ -31,14 +31,31 @@ public final class InternalNettyChannelBuilder { /** * Checks authority upon channel construction. The purpose of this interface is to raise the * visibility of {@link NettyChannelBuilder.OverrideAuthorityChecker}. + * @deprecated To be removed, use {@link #disableCheckAuthority(NettyChannelBuilder builder)} to + * disable authority check. */ + @Deprecated public interface OverrideAuthorityChecker extends NettyChannelBuilder.OverrideAuthorityChecker {} + /** + * Overrides authority checker. + * @deprecated To be removed, use {@link #disableCheckAuthority(NettyChannelBuilder builder)} to + * disable authority check. + */ + @Deprecated public static void overrideAuthorityChecker( NettyChannelBuilder channelBuilder, OverrideAuthorityChecker authorityChecker) { channelBuilder.overrideAuthorityChecker(authorityChecker); } + public static void disableCheckAuthority(NettyChannelBuilder builder) { + builder.disableCheckAuthority(); + } + + public static void enableCheckAuthority(NettyChannelBuilder builder) { + builder.enableCheckAuthority(); + } + /** A class that provides a Netty handler to control protocol negotiation. */ public interface ProtocolNegotiatorFactory extends NettyChannelBuilder.ProtocolNegotiatorFactory { @@ -68,6 +85,10 @@ public static void setStatsRecordStartedRpcs(NettyChannelBuilder builder, boolea builder.setStatsRecordStartedRpcs(value); } + public static void setStatsRecordFinishedRpcs(NettyChannelBuilder builder, boolean value) { + builder.setStatsRecordFinishedRpcs(value); + } + public static void setStatsRecordRealTimeMetrics(NettyChannelBuilder builder, boolean value) { builder.setStatsRecordRealTimeMetrics(value); } diff --git a/netty/src/main/java/io/grpc/netty/NettyChannelBuilder.java b/netty/src/main/java/io/grpc/netty/NettyChannelBuilder.java index 59e86a117da..43d6b96d507 100644 --- a/netty/src/main/java/io/grpc/netty/NettyChannelBuilder.java +++ b/netty/src/main/java/io/grpc/netty/NettyChannelBuilder.java @@ -28,15 +28,19 @@ import io.grpc.ChannelLogger; import io.grpc.EquivalentAddressGroup; import io.grpc.ExperimentalApi; +import io.grpc.ForwardingChannelBuilder; import io.grpc.HttpConnectProxiedSocketAddress; import io.grpc.Internal; -import io.grpc.internal.AbstractManagedChannelImplBuilder; +import io.grpc.ManagedChannelBuilder; import io.grpc.internal.AtomicBackoff; import io.grpc.internal.ClientTransportFactory; import io.grpc.internal.ConnectionClientTransport; import io.grpc.internal.FixedObjectPool; import io.grpc.internal.GrpcUtil; import io.grpc.internal.KeepAliveManager; +import io.grpc.internal.ManagedChannelImplBuilder; +import io.grpc.internal.ManagedChannelImplBuilder.ChannelBuilderDefaultPortProvider; +import io.grpc.internal.ManagedChannelImplBuilder.ClientTransportFactoryBuilder; import io.grpc.internal.ObjectPool; import io.grpc.internal.SharedResourcePool; import io.grpc.internal.TransportTracer; @@ -63,8 +67,7 @@ */ @ExperimentalApi("https://github.com/grpc/grpc-java/issues/1784") @CanIgnoreReturnValue -public final class NettyChannelBuilder - extends AbstractManagedChannelImplBuilder { +public final class NettyChannelBuilder extends ForwardingChannelBuilder { // 1MiB. public static final int DEFAULT_FLOW_CONTROL_WINDOW = 1024 * 1024; @@ -85,16 +88,16 @@ public final class NettyChannelBuilder DEFAULT_AUTO_FLOW_CONTROL = Boolean.parseBoolean(autoFlowControl); } - private final Map, Object> channelOptions = - new HashMap<>(); - + private final ManagedChannelImplBuilder managedChannelImplBuilder; + private TransportTracer.Factory transportTracerFactory = TransportTracer.getDefaultFactory(); + private final Map, Object> channelOptions = new HashMap<>(); private NegotiationType negotiationType = NegotiationType.TLS; - private OverrideAuthorityChecker authorityChecker; private ChannelFactory channelFactory = DEFAULT_CHANNEL_FACTORY; private ObjectPool eventLoopGroupPool = DEFAULT_EVENT_LOOP_GROUP_POOL; private SslContext sslContext; private boolean autoFlowControl = DEFAULT_AUTO_FLOW_CONTROL; private int flowControlWindow = DEFAULT_FLOW_CONTROL_WINDOW; + private int maxInboundMessageSize = GrpcUtil.DEFAULT_MAX_MESSAGE_SIZE; private int maxHeaderListSize = GrpcUtil.DEFAULT_MAX_HEADER_LIST_SIZE; private long keepAliveTimeNanos = KEEPALIVE_TIME_NANOS_DISABLED; private long keepAliveTimeoutNanos = DEFAULT_KEEPALIVE_TIMEOUT_NANOS; @@ -142,14 +145,39 @@ public static NettyChannelBuilder forTarget(String target) { this(GrpcUtil.authorityFromHostAndPort(host, port)); } + private final class NettyChannelTransportFactoryBuilder implements ClientTransportFactoryBuilder { + @Override + public ClientTransportFactory buildClientTransportFactory() { + return buildTransportFactory(); + } + } + + private final class NettyChannelDefaultPortProvider implements ChannelBuilderDefaultPortProvider { + @Override + public int getDefaultPort() { + return NettyChannelBuilder.this.getDefaultPort(); + } + } + @CheckReturnValue NettyChannelBuilder(String target) { - super(target); + managedChannelImplBuilder = new ManagedChannelImplBuilder(target, + new NettyChannelTransportFactoryBuilder(), + new NettyChannelDefaultPortProvider()); } @CheckReturnValue NettyChannelBuilder(SocketAddress address) { - super(address, getAuthorityFromAddress(address)); + managedChannelImplBuilder = new ManagedChannelImplBuilder(address, + getAuthorityFromAddress(address), + new NettyChannelTransportFactoryBuilder(), + new NettyChannelDefaultPortProvider()); + } + + @Internal + @Override + protected ManagedChannelBuilder delegate() { + return managedChannelImplBuilder; } @CheckReturnValue @@ -408,10 +436,20 @@ public SocketAddress createSocketAddress( } } + /** + * Sets the maximum message size allowed for a single gRPC frame. If an inbound messages larger + * than this limit is received it will not be processed and the RPC will fail with + * RESOURCE_EXHAUSTED. + */ @Override + public NettyChannelBuilder maxInboundMessageSize(int max) { + checkArgument(max >= 0, "negative max"); + maxInboundMessageSize = max; + return this; + } + @CheckReturnValue - @Internal - protected ClientTransportFactory buildTransportFactory() { + ClientTransportFactory buildTransportFactory() { assertEventLoopAndChannelType(); ProtocolNegotiator negotiator; @@ -427,12 +465,12 @@ protected ClientTransportFactory buildTransportFactory() { } } negotiator = createProtocolNegotiatorByType(negotiationType, localSslContext, - this.getOffloadExecutorPool()); + this.managedChannelImplBuilder.getOffloadExecutorPool()); } return new NettyTransportFactory( negotiator, channelFactory, channelOptions, - eventLoopGroupPool, autoFlowControl, flowControlWindow, maxInboundMessageSize(), + eventLoopGroupPool, autoFlowControl, flowControlWindow, maxInboundMessageSize, maxHeaderListSize, keepAliveTimeNanos, keepAliveTimeoutNanos, keepAliveWithoutCalls, transportTracerFactory, localSocketPicker, useGetForSafeMethods); } @@ -448,9 +486,8 @@ void assertEventLoopAndChannelType() { "Both EventLoopGroup and ChannelType should be provided or neither should be"); } - @Override @CheckReturnValue - protected int getDefaultPort() { + int getDefaultPort() { switch (negotiationType) { case PLAINTEXT: case PLAINTEXT_UPGRADE: @@ -462,10 +499,6 @@ protected int getDefaultPort() { } } - void overrideAuthorityChecker(@Nullable OverrideAuthorityChecker authorityChecker) { - this.authorityChecker = authorityChecker; - } - @VisibleForTesting @CheckReturnValue static ProtocolNegotiator createProtocolNegotiatorByType( @@ -484,19 +517,22 @@ static ProtocolNegotiator createProtocolNegotiatorByType( } } - @CheckReturnValue - interface OverrideAuthorityChecker { - String checkAuthority(String authority); + @Deprecated + interface OverrideAuthorityChecker extends ManagedChannelImplBuilder.OverrideAuthorityChecker {} + + @Deprecated + void overrideAuthorityChecker(@Nullable OverrideAuthorityChecker authorityChecker) { + this.managedChannelImplBuilder.overrideAuthorityChecker(authorityChecker); } - @Override - @CheckReturnValue - @Internal - protected String checkAuthority(String authority) { - if (authorityChecker != null) { - return authorityChecker.checkAuthority(authority); - } - return super.checkAuthority(authority); + NettyChannelBuilder disableCheckAuthority() { + this.managedChannelImplBuilder.disableCheckAuthority(); + return this; + } + + NettyChannelBuilder enableCheckAuthority() { + this.managedChannelImplBuilder.enableCheckAuthority(); + return this; } void protocolNegotiatorFactory(ProtocolNegotiatorFactory protocolNegotiatorFactory) { @@ -504,24 +540,24 @@ void protocolNegotiatorFactory(ProtocolNegotiatorFactory protocolNegotiatorFacto = checkNotNull(protocolNegotiatorFactory, "protocolNegotiatorFactory"); } - @Override - protected void setTracingEnabled(boolean value) { - super.setTracingEnabled(value); + void setTracingEnabled(boolean value) { + this.managedChannelImplBuilder.setTracingEnabled(value); } - @Override - protected void setStatsEnabled(boolean value) { - super.setStatsEnabled(value); + void setStatsEnabled(boolean value) { + this.managedChannelImplBuilder.setStatsEnabled(value); } - @Override - protected void setStatsRecordStartedRpcs(boolean value) { - super.setStatsRecordStartedRpcs(value); + void setStatsRecordStartedRpcs(boolean value) { + this.managedChannelImplBuilder.setStatsRecordStartedRpcs(value); } - @Override - protected void setStatsRecordRealTimeMetrics(boolean value) { - super.setStatsRecordRealTimeMetrics(value); + void setStatsRecordFinishedRpcs(boolean value) { + this.managedChannelImplBuilder.setStatsRecordFinishedRpcs(value); + } + + void setStatsRecordRealTimeMetrics(boolean value) { + this.managedChannelImplBuilder.setStatsRecordRealTimeMetrics(value); } @VisibleForTesting diff --git a/netty/src/test/java/io/grpc/netty/NettyChannelBuilderTest.java b/netty/src/test/java/io/grpc/netty/NettyChannelBuilderTest.java index 3621e6e2454..9a96d73e5c5 100644 --- a/netty/src/test/java/io/grpc/netty/NettyChannelBuilderTest.java +++ b/netty/src/test/java/io/grpc/netty/NettyChannelBuilderTest.java @@ -22,7 +22,7 @@ import static org.mockito.Mockito.mock; import io.grpc.ManagedChannel; -import io.grpc.netty.InternalNettyChannelBuilder.OverrideAuthorityChecker; +import io.grpc.internal.GrpcUtil; import io.grpc.netty.NettyTestUtil.TrackingObjectPoolForTest; import io.netty.channel.Channel; import io.netty.channel.ChannelFactory; @@ -44,7 +44,7 @@ public class NettyChannelBuilderTest { @Rule public final ExpectedException thrown = ExpectedException.none(); private final SslContext noSslContext = null; - + private void shutdown(ManagedChannel mc) throws Exception { mc.shutdownNow(); assertTrue(mc.awaitTermination(1, TimeUnit.SECONDS)); @@ -92,14 +92,35 @@ private void overrideAuthorityIsReadableHelper(NettyChannelBuilder builder, } @Test + @Deprecated public void overrideAllowsInvalidAuthority() { NettyChannelBuilder builder = new NettyChannelBuilder(new SocketAddress(){}); - InternalNettyChannelBuilder.overrideAuthorityChecker(builder, new OverrideAuthorityChecker() { - @Override - public String checkAuthority(String authority) { - return authority; - } - }); + InternalNettyChannelBuilder.overrideAuthorityChecker(builder, + new io.grpc.netty.InternalNettyChannelBuilder.OverrideAuthorityChecker() { + @Override + public String checkAuthority(String authority) { + return authority; + } + }); + Object unused = builder.overrideAuthority("[invalidauthority") + .negotiationType(NegotiationType.PLAINTEXT) + .buildTransportFactory(); + } + + @Test + @Deprecated + public void overrideFailsInvalidAuthority() { + NettyChannelBuilder builder = new NettyChannelBuilder(new SocketAddress(){}); + InternalNettyChannelBuilder.overrideAuthorityChecker(builder, + new io.grpc.netty.InternalNettyChannelBuilder.OverrideAuthorityChecker() { + @Override + public String checkAuthority(String authority) { + return GrpcUtil.checkAuthority(authority); + } + }); + + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("Invalid authority:"); Object unused = builder.overrideAuthority("[invalidauthority") .negotiationType(NegotiationType.PLAINTEXT) .buildTransportFactory(); @@ -115,6 +136,27 @@ public void failOverrideInvalidAuthority() { builder.overrideAuthority("[invalidauthority"); } + @Test + public void disableCheckAuthorityAllowsInvalidAuthority() { + NettyChannelBuilder builder = new NettyChannelBuilder(new SocketAddress(){}) + .disableCheckAuthority(); + + Object unused = builder.overrideAuthority("[invalidauthority") + .negotiationType(NegotiationType.PLAINTEXT) + .buildTransportFactory(); + } + + @Test + public void enableCheckAuthorityFailOverrideInvalidAuthority() { + NettyChannelBuilder builder = new NettyChannelBuilder(new SocketAddress(){}) + .disableCheckAuthority() + .enableCheckAuthority(); + + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("Invalid authority:"); + builder.overrideAuthority("[invalidauthority"); + } + @Test public void failInvalidAuthority() { thrown.expect(IllegalArgumentException.class); From 5d1304c33c668f88c55aa26396ba10efc7836b3b Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Wed, 2 Sep 2020 11:39:08 -0400 Subject: [PATCH 13/86] okhttp: OkHttpChannelBuilder extends a public API class --- .../integration/TestServiceClient.java | 51 +++++++------- .../testing/integration/Http2OkHttpTest.java | 3 +- .../okhttp/InternalOkHttpChannelBuilder.java | 33 +++++++++ .../io/grpc/okhttp/OkHttpChannelBuilder.java | 70 ++++++++++++++++--- .../grpc/okhttp/OkHttpChannelBuilderTest.java | 24 ++++--- 5 files changed, 136 insertions(+), 45 deletions(-) create mode 100644 okhttp/src/main/java/io/grpc/okhttp/InternalOkHttpChannelBuilder.java diff --git a/interop-testing/src/main/java/io/grpc/testing/integration/TestServiceClient.java b/interop-testing/src/main/java/io/grpc/testing/integration/TestServiceClient.java index c06a525c85b..a0a484a64a7 100644 --- a/interop-testing/src/main/java/io/grpc/testing/integration/TestServiceClient.java +++ b/interop-testing/src/main/java/io/grpc/testing/integration/TestServiceClient.java @@ -23,13 +23,13 @@ import io.grpc.alts.AltsChannelBuilder; import io.grpc.alts.ComputeEngineChannelBuilder; import io.grpc.alts.GoogleDefaultChannelBuilder; -import io.grpc.internal.AbstractManagedChannelImplBuilder; import io.grpc.internal.GrpcUtil; import io.grpc.internal.testing.TestUtils; import io.grpc.netty.GrpcSslContexts; import io.grpc.netty.InternalNettyChannelBuilder; import io.grpc.netty.NegotiationType; import io.grpc.netty.NettyChannelBuilder; +import io.grpc.okhttp.InternalOkHttpChannelBuilder; import io.grpc.okhttp.OkHttpChannelBuilder; import io.grpc.okhttp.internal.Platform; import io.netty.handler.ssl.SslContext; @@ -403,7 +403,7 @@ protected ManagedChannelBuilder createChannelBuilder() { if (useAlts) { return AltsChannelBuilder.forAddress(serverHost, serverPort); } - AbstractManagedChannelImplBuilder builder; + if (!useOkHttp) { SslContext sslContext = null; if (useTestCa) { @@ -429,34 +429,33 @@ protected ManagedChannelBuilder createChannelBuilder() { // Disable the default census stats interceptor, use testing interceptor instead. InternalNettyChannelBuilder.setStatsEnabled(nettyBuilder, false); return nettyBuilder.intercept(createCensusStatsClientInterceptor()); - } else { - OkHttpChannelBuilder okBuilder = OkHttpChannelBuilder.forAddress(serverHost, serverPort); - if (serverHostOverride != null) { - // Force the hostname to match the cert the server uses. - okBuilder.overrideAuthority( - GrpcUtil.authorityFromHostAndPort(serverHostOverride, serverPort)); - } - if (useTls) { - if (useTestCa) { - try { - SSLSocketFactory factory = TestUtils.newSslSocketFactoryForCa( - Platform.get().getProvider(), TestUtils.loadCert("ca.pem")); - okBuilder.sslSocketFactory(factory); - } catch (Exception e) { - throw new RuntimeException(e); - } + } + + OkHttpChannelBuilder okBuilder = OkHttpChannelBuilder.forAddress(serverHost, serverPort); + if (serverHostOverride != null) { + // Force the hostname to match the cert the server uses. + okBuilder.overrideAuthority( + GrpcUtil.authorityFromHostAndPort(serverHostOverride, serverPort)); + } + if (useTls) { + if (useTestCa) { + try { + SSLSocketFactory factory = TestUtils.newSslSocketFactoryForCa( + Platform.get().getProvider(), TestUtils.loadCert("ca.pem")); + okBuilder.sslSocketFactory(factory); + } catch (Exception e) { + throw new RuntimeException(e); } - } else { - okBuilder.usePlaintext(); - } - if (fullStreamDecompression) { - okBuilder.enableFullStreamDecompression(); } - builder = okBuilder; + } else { + okBuilder.usePlaintext(); + } + if (fullStreamDecompression) { + okBuilder.enableFullStreamDecompression(); } // Disable the default census stats interceptor, use testing interceptor instead. - io.grpc.internal.TestingAccessor.setStatsEnabled(builder, false); - return builder.intercept(createCensusStatsClientInterceptor()); + InternalOkHttpChannelBuilder.setStatsEnabled(okBuilder, false); + return okBuilder.intercept(createCensusStatsClientInterceptor()); } @Override diff --git a/interop-testing/src/test/java/io/grpc/testing/integration/Http2OkHttpTest.java b/interop-testing/src/test/java/io/grpc/testing/integration/Http2OkHttpTest.java index 041c7cafaa6..fc6e600789d 100644 --- a/interop-testing/src/test/java/io/grpc/testing/integration/Http2OkHttpTest.java +++ b/interop-testing/src/test/java/io/grpc/testing/integration/Http2OkHttpTest.java @@ -29,6 +29,7 @@ import io.grpc.internal.testing.TestUtils; import io.grpc.netty.GrpcSslContexts; import io.grpc.netty.NettyServerBuilder; +import io.grpc.okhttp.InternalOkHttpChannelBuilder; import io.grpc.okhttp.OkHttpChannelBuilder; import io.grpc.okhttp.internal.Platform; import io.grpc.stub.StreamObserver; @@ -102,7 +103,7 @@ protected OkHttpChannelBuilder createChannelBuilder() { throw new RuntimeException(e); } // Disable the default census stats interceptor, use testing interceptor instead. - io.grpc.internal.TestingAccessor.setStatsEnabled(builder, false); + InternalOkHttpChannelBuilder.setStatsEnabled(builder, false); return builder.intercept(createCensusStatsClientInterceptor()); } diff --git a/okhttp/src/main/java/io/grpc/okhttp/InternalOkHttpChannelBuilder.java b/okhttp/src/main/java/io/grpc/okhttp/InternalOkHttpChannelBuilder.java new file mode 100644 index 00000000000..d328efd7145 --- /dev/null +++ b/okhttp/src/main/java/io/grpc/okhttp/InternalOkHttpChannelBuilder.java @@ -0,0 +1,33 @@ +/* + * Copyright 2020 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.okhttp; + +import io.grpc.Internal; + +/** + * Internal {@link OkHttpChannelBuilder} accessor. This is intended for usage internal to the gRPC + * team. If you *really* think you need to use this, contact the gRPC team first. + */ +@Internal +public final class InternalOkHttpChannelBuilder { + + public static void setStatsEnabled(OkHttpChannelBuilder builder, boolean value) { + builder.setStatsEnabled(value); + } + + private InternalOkHttpChannelBuilder() {} +} diff --git a/okhttp/src/main/java/io/grpc/okhttp/OkHttpChannelBuilder.java b/okhttp/src/main/java/io/grpc/okhttp/OkHttpChannelBuilder.java index 26a76f2c8c4..a7759127e62 100644 --- a/okhttp/src/main/java/io/grpc/okhttp/OkHttpChannelBuilder.java +++ b/okhttp/src/main/java/io/grpc/okhttp/OkHttpChannelBuilder.java @@ -24,13 +24,17 @@ import com.google.common.base.Preconditions; import io.grpc.ChannelLogger; import io.grpc.ExperimentalApi; +import io.grpc.ForwardingChannelBuilder; import io.grpc.Internal; -import io.grpc.internal.AbstractManagedChannelImplBuilder; +import io.grpc.ManagedChannelBuilder; import io.grpc.internal.AtomicBackoff; import io.grpc.internal.ClientTransportFactory; import io.grpc.internal.ConnectionClientTransport; import io.grpc.internal.GrpcUtil; import io.grpc.internal.KeepAliveManager; +import io.grpc.internal.ManagedChannelImplBuilder; +import io.grpc.internal.ManagedChannelImplBuilder.ChannelBuilderDefaultPortProvider; +import io.grpc.internal.ManagedChannelImplBuilder.ClientTransportFactoryBuilder; import io.grpc.internal.SharedResourceHolder; import io.grpc.internal.SharedResourceHolder.Resource; import io.grpc.internal.TransportTracer; @@ -54,10 +58,12 @@ /** Convenience class for building channels with the OkHttp transport. */ @ExperimentalApi("https://github.com/grpc/grpc-java/issues/1785") -public class OkHttpChannelBuilder extends - AbstractManagedChannelImplBuilder { +public class OkHttpChannelBuilder extends ForwardingChannelBuilder { public static final int DEFAULT_FLOW_CONTROL_WINDOW = 65535; + private final ManagedChannelImplBuilder managedChannelImplBuilder; + private TransportTracer.Factory transportTracerFactory = TransportTracer.getDefaultFactory(); + /** Identifies the negotiation used for starting up HTTP/2. */ private enum NegotiationType { @@ -127,6 +133,7 @@ public static OkHttpChannelBuilder forTarget(String target) { private long keepAliveTimeoutNanos = DEFAULT_KEEPALIVE_TIMEOUT_NANOS; private int flowControlWindow = DEFAULT_FLOW_CONTROL_WINDOW; private boolean keepAliveWithoutCalls; + private int maxInboundMessageSize = GrpcUtil.DEFAULT_MAX_MESSAGE_SIZE; private int maxInboundMetadataSize = Integer.MAX_VALUE; /** @@ -140,7 +147,29 @@ protected OkHttpChannelBuilder(String host, int port) { } private OkHttpChannelBuilder(String target) { - super(target); + final class OkHttpChannelTransportFactoryBuilder implements ClientTransportFactoryBuilder { + @Override + public ClientTransportFactory buildClientTransportFactory() { + return buildTransportFactory(); + } + } + + final class OkHttpChannelDefaultPortProvider implements ChannelBuilderDefaultPortProvider { + @Override + public int getDefaultPort() { + return OkHttpChannelBuilder.this.getDefaultPort(); + } + } + + managedChannelImplBuilder = new ManagedChannelImplBuilder(target, + new OkHttpChannelTransportFactoryBuilder(), + new OkHttpChannelDefaultPortProvider()); + } + + @Internal + @Override + protected final ManagedChannelBuilder delegate() { + return managedChannelImplBuilder; } @VisibleForTesting @@ -363,9 +392,19 @@ public OkHttpChannelBuilder maxInboundMetadataSize(int bytes) { return this; } + /** + * Sets the maximum message size allowed for a single gRPC frame. If an inbound messages + * larger than this limit is received it will not be processed and the RPC will fail with + * RESOURCE_EXHAUSTED. + */ @Override - @Internal - protected final ClientTransportFactory buildTransportFactory() { + public final OkHttpChannelBuilder maxInboundMessageSize(int max) { + Preconditions.checkArgument(max >= 0, "negative max"); + maxInboundMessageSize = max; + return this; + } + + final ClientTransportFactory buildTransportFactory() { boolean enableKeepAlive = keepAliveTimeNanos != KEEPALIVE_TIME_NANOS_DISABLED; return new OkHttpTransportFactory( transportExecutor, @@ -374,7 +413,7 @@ protected final ClientTransportFactory buildTransportFactory() { createSslSocketFactory(), hostnameVerifier, connectionSpec, - maxInboundMessageSize(), + maxInboundMessageSize, enableKeepAlive, keepAliveTimeNanos, keepAliveTimeoutNanos, @@ -385,8 +424,17 @@ protected final ClientTransportFactory buildTransportFactory() { useGetForSafeMethods); } - @Override - protected int getDefaultPort() { + final OkHttpChannelBuilder disableCheckAuthority() { + this.managedChannelImplBuilder.disableCheckAuthority(); + return this; + } + + final OkHttpChannelBuilder enableCheckAuthority() { + this.managedChannelImplBuilder.enableCheckAuthority(); + return this; + } + + final int getDefaultPort() { switch (negotiationType) { case PLAINTEXT: return GrpcUtil.DEFAULT_PORT_PLAINTEXT; @@ -397,6 +445,10 @@ protected int getDefaultPort() { } } + final void setStatsEnabled(boolean value) { + this.managedChannelImplBuilder.setStatsEnabled(value); + } + @VisibleForTesting @Nullable SSLSocketFactory createSslSocketFactory() { diff --git a/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java b/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java index 512107eabca..4aa1d1aa53c 100644 --- a/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java +++ b/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java @@ -72,20 +72,26 @@ private void overrideAuthorityIsReadableHelper(OkHttpChannelBuilder builder, } @Test - public void overrideAllowsInvalidAuthority() { - OkHttpChannelBuilder builder = new OkHttpChannelBuilder("good", 1234) { - @Override - protected String checkAuthority(String authority) { - return authority; - } - }; + public void failOverrideInvalidAuthority() { + OkHttpChannelBuilder builder = new OkHttpChannelBuilder("good", 1234); + + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("Invalid authority:"); + builder.overrideAuthority("[invalidauthority"); + } + @Test + public void disableCheckAuthorityAllowsInvalidAuthority() { + OkHttpChannelBuilder builder = new OkHttpChannelBuilder("good", 1234) + .disableCheckAuthority(); builder.overrideAuthority("[invalidauthority").usePlaintext().buildTransportFactory(); } @Test - public void failOverrideInvalidAuthority() { - OkHttpChannelBuilder builder = new OkHttpChannelBuilder("good", 1234); + public void enableCheckAuthorityFailOverrideInvalidAuthority() { + OkHttpChannelBuilder builder = new OkHttpChannelBuilder("good", 1234) + .disableCheckAuthority() + .enableCheckAuthority(); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Invalid authority:"); From e335cb3618042df3819e23dd20ac3a8a5356a6e6 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Wed, 2 Sep 2020 12:06:07 -0400 Subject: [PATCH 14/86] testing: remove channel setStatsEnabled accessor hack --- .../src/main/java/io/grpc/internal/TestingAccessor.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/testing/src/main/java/io/grpc/internal/TestingAccessor.java b/testing/src/main/java/io/grpc/internal/TestingAccessor.java index d5b00e025fc..92dc114d9c6 100644 --- a/testing/src/main/java/io/grpc/internal/TestingAccessor.java +++ b/testing/src/main/java/io/grpc/internal/TestingAccessor.java @@ -20,13 +20,6 @@ * Test helper that allows accessing package-private stuff. */ public final class TestingAccessor { - /** - * Disable or enable client side census stats features. - */ - public static void setStatsEnabled( - AbstractManagedChannelImplBuilder builder, boolean statsEnabled) { - builder.setStatsEnabled(statsEnabled); - } /** * Disable or enable server side census stats features. From c29ad76daea0d17637fefcab4dbfd9ef8cb84ebc Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Wed, 2 Sep 2020 14:11:34 -0400 Subject: [PATCH 15/86] Explain test value for flow control window --- .../test/java/io/grpc/internal/AbstractTransportTest.java | 6 ++++++ .../grpc/testing/integration/NettyClientInteropServlet.java | 3 ++- .../io/grpc/testing/integration/AbstractInteropTest.java | 6 ++++++ .../java/io/grpc/testing/integration/TestServiceClient.java | 2 +- .../testing/integration/Http2NettyLocalChannelTest.java | 4 ++-- .../java/io/grpc/testing/integration/Http2NettyTest.java | 4 ++-- .../java/io/grpc/testing/integration/Http2OkHttpTest.java | 2 +- netty/src/test/java/io/grpc/netty/NettyTransportTest.java | 6 +++--- .../src/test/java/io/grpc/okhttp/OkHttpTransportTest.java | 4 ++-- 9 files changed, 25 insertions(+), 12 deletions(-) diff --git a/core/src/test/java/io/grpc/internal/AbstractTransportTest.java b/core/src/test/java/io/grpc/internal/AbstractTransportTest.java index cca994c105a..aa9f269de20 100644 --- a/core/src/test/java/io/grpc/internal/AbstractTransportTest.java +++ b/core/src/test/java/io/grpc/internal/AbstractTransportTest.java @@ -89,6 +89,12 @@ /** Standard unit tests for {@link ClientTransport}s and {@link ServerTransport}s. */ @RunWith(JUnit4.class) public abstract class AbstractTransportTest { + /** + * Use a small flow control to help detect flow control bugs. Don't use 64KiB to test + * SETTINGS/WINDOW_UPDATE exchange. + */ + public static final int TEST_FLOW_CONTROL_WINDOW = 65 * 1024; + private static final int TIMEOUT_MS = 5000; private static final Attributes.Key ADDITIONAL_TRANSPORT_ATTR_KEY = diff --git a/gae-interop-testing/gae-jdk8/src/main/java/io/grpc/testing/integration/NettyClientInteropServlet.java b/gae-interop-testing/gae-jdk8/src/main/java/io/grpc/testing/integration/NettyClientInteropServlet.java index b42f9aef315..2a9c195841e 100644 --- a/gae-interop-testing/gae-jdk8/src/main/java/io/grpc/testing/integration/NettyClientInteropServlet.java +++ b/gae-interop-testing/gae-jdk8/src/main/java/io/grpc/testing/integration/NettyClientInteropServlet.java @@ -134,7 +134,8 @@ protected ManagedChannelBuilder createChannelBuilder() { ManagedChannelBuilder.forTarget(INTEROP_TEST_ADDRESS) .maxInboundMessageSize(AbstractInteropTest.MAX_MESSAGE_SIZE); assertTrue(builder instanceof NettyChannelBuilder); - ((NettyChannelBuilder) builder).flowControlWindow(65 * 1024); + ((NettyChannelBuilder) builder) + .flowControlWindow(AbstractInteropTest.TEST_FLOW_CONTROL_WINDOW); return builder; } diff --git a/interop-testing/src/main/java/io/grpc/testing/integration/AbstractInteropTest.java b/interop-testing/src/main/java/io/grpc/testing/integration/AbstractInteropTest.java index 46122d5f256..643d984bad8 100644 --- a/interop-testing/src/main/java/io/grpc/testing/integration/AbstractInteropTest.java +++ b/interop-testing/src/main/java/io/grpc/testing/integration/AbstractInteropTest.java @@ -148,6 +148,12 @@ public abstract class AbstractInteropTest { /** Must be at least {@link #unaryPayloadLength()}, plus some to account for encoding overhead. */ public static final int MAX_MESSAGE_SIZE = 16 * 1024 * 1024; + /** + * Use a small flow control to help detect flow control bugs. Don't use 64KiB to test + * SETTINGS/WINDOW_UPDATE exchange. + */ + public static final int TEST_FLOW_CONTROL_WINDOW = 65 * 1024; + private static final FakeTagger tagger = new FakeTagger(); private static final FakeTagContextBinarySerializer tagContextBinarySerializer = new FakeTagContextBinarySerializer(); diff --git a/interop-testing/src/main/java/io/grpc/testing/integration/TestServiceClient.java b/interop-testing/src/main/java/io/grpc/testing/integration/TestServiceClient.java index a0a484a64a7..ff4c6ca06be 100644 --- a/interop-testing/src/main/java/io/grpc/testing/integration/TestServiceClient.java +++ b/interop-testing/src/main/java/io/grpc/testing/integration/TestServiceClient.java @@ -416,7 +416,7 @@ protected ManagedChannelBuilder createChannelBuilder() { } NettyChannelBuilder nettyBuilder = NettyChannelBuilder.forAddress(serverHost, serverPort) - .flowControlWindow(65 * 1024) + .flowControlWindow(AbstractInteropTest.TEST_FLOW_CONTROL_WINDOW) .negotiationType(useTls ? NegotiationType.TLS : (useH2cUpgrade ? NegotiationType.PLAINTEXT_UPGRADE : NegotiationType.PLAINTEXT)) .sslContext(sslContext); diff --git a/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyLocalChannelTest.java b/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyLocalChannelTest.java index 8ed7dc76900..4494d620b1e 100644 --- a/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyLocalChannelTest.java +++ b/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyLocalChannelTest.java @@ -41,7 +41,7 @@ public class Http2NettyLocalChannelTest extends AbstractInteropTest { protected AbstractServerImplBuilder getServerBuilder() { return NettyServerBuilder .forAddress(new LocalAddress("in-process-1")) - .flowControlWindow(65 * 1024) + .flowControlWindow(AbstractInteropTest.TEST_FLOW_CONTROL_WINDOW) .maxInboundMessageSize(AbstractInteropTest.MAX_MESSAGE_SIZE) .channelType(LocalServerChannel.class) .workerEventLoopGroup(eventLoopGroup) @@ -55,7 +55,7 @@ protected NettyChannelBuilder createChannelBuilder() { .negotiationType(NegotiationType.PLAINTEXT) .channelType(LocalChannel.class) .eventLoopGroup(eventLoopGroup) - .flowControlWindow(65 * 1024) + .flowControlWindow(AbstractInteropTest.TEST_FLOW_CONTROL_WINDOW) .maxInboundMessageSize(AbstractInteropTest.MAX_MESSAGE_SIZE); // Disable the default census stats interceptor, use testing interceptor instead. InternalNettyChannelBuilder.setStatsEnabled(builder, false); diff --git a/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyTest.java b/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyTest.java index 353180cbafb..9053a1fff64 100644 --- a/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyTest.java +++ b/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyTest.java @@ -45,7 +45,7 @@ protected AbstractServerImplBuilder getServerBuilder() { // Starts the server with HTTPS. try { return NettyServerBuilder.forPort(0) - .flowControlWindow(65 * 1024) + .flowControlWindow(AbstractInteropTest.TEST_FLOW_CONTROL_WINDOW) .maxInboundMessageSize(AbstractInteropTest.MAX_MESSAGE_SIZE) .sslContext(GrpcSslContexts .forServer(TestUtils.loadCert("server1.pem"), TestUtils.loadCert("server1.key")) @@ -63,7 +63,7 @@ protected NettyChannelBuilder createChannelBuilder() { try { NettyChannelBuilder builder = NettyChannelBuilder .forAddress(TestUtils.testServerAddress((InetSocketAddress) getListenAddress())) - .flowControlWindow(65 * 1024) + .flowControlWindow(AbstractInteropTest.TEST_FLOW_CONTROL_WINDOW) .maxInboundMessageSize(AbstractInteropTest.MAX_MESSAGE_SIZE) .sslContext(GrpcSslContexts .forClient() diff --git a/interop-testing/src/test/java/io/grpc/testing/integration/Http2OkHttpTest.java b/interop-testing/src/test/java/io/grpc/testing/integration/Http2OkHttpTest.java index fc6e600789d..ea4ba5877ce 100644 --- a/interop-testing/src/test/java/io/grpc/testing/integration/Http2OkHttpTest.java +++ b/interop-testing/src/test/java/io/grpc/testing/integration/Http2OkHttpTest.java @@ -78,7 +78,7 @@ protected AbstractServerImplBuilder getServerBuilder() { GrpcSslContexts.configure(contextBuilder, sslProvider); contextBuilder.ciphers(TestUtils.preferredTestCiphers(), SupportedCipherSuiteFilter.INSTANCE); return NettyServerBuilder.forPort(0) - .flowControlWindow(65 * 1024) + .flowControlWindow(AbstractInteropTest.TEST_FLOW_CONTROL_WINDOW) .maxInboundMessageSize(AbstractInteropTest.MAX_MESSAGE_SIZE) .sslContext(contextBuilder.build()); } catch (IOException ex) { diff --git a/netty/src/test/java/io/grpc/netty/NettyTransportTest.java b/netty/src/test/java/io/grpc/netty/NettyTransportTest.java index 182fd81d9d8..7280ba64946 100644 --- a/netty/src/test/java/io/grpc/netty/NettyTransportTest.java +++ b/netty/src/test/java/io/grpc/netty/NettyTransportTest.java @@ -47,7 +47,7 @@ public class NettyTransportTest extends AbstractTransportTest { private final ClientTransportFactory clientFactory = NettyChannelBuilder // Although specified here, address is ignored because we never call build. .forAddress("localhost", 0) - .flowControlWindow(65 * 1024) + .flowControlWindow(AbstractTransportTest.TEST_FLOW_CONTROL_WINDOW) .negotiationType(NegotiationType.PLAINTEXT) .setTransportTracerFactory(fakeClockTransportTracer) .buildTransportFactory(); @@ -67,7 +67,7 @@ protected List newServer( List streamTracerFactories) { return NettyServerBuilder .forAddress(new InetSocketAddress("localhost", 0)) - .flowControlWindow(65 * 1024) + .flowControlWindow(AbstractTransportTest.TEST_FLOW_CONTROL_WINDOW) .setTransportTracerFactory(fakeClockTransportTracer) .buildTransportServers(streamTracerFactories); } @@ -77,7 +77,7 @@ protected List newServer( int port, List streamTracerFactories) { return NettyServerBuilder .forAddress(new InetSocketAddress("localhost", port)) - .flowControlWindow(65 * 1024) + .flowControlWindow(AbstractTransportTest.TEST_FLOW_CONTROL_WINDOW) .setTransportTracerFactory(fakeClockTransportTracer) .buildTransportServers(streamTracerFactories); } diff --git a/okhttp/src/test/java/io/grpc/okhttp/OkHttpTransportTest.java b/okhttp/src/test/java/io/grpc/okhttp/OkHttpTransportTest.java index ac6b1122b90..8e99b549a31 100644 --- a/okhttp/src/test/java/io/grpc/okhttp/OkHttpTransportTest.java +++ b/okhttp/src/test/java/io/grpc/okhttp/OkHttpTransportTest.java @@ -56,7 +56,7 @@ protected List newServer( return AccessProtectedHack.serverBuilderBuildTransportServer( NettyServerBuilder .forPort(0) - .flowControlWindow(65 * 1024), + .flowControlWindow(AbstractTransportTest.TEST_FLOW_CONTROL_WINDOW), streamTracerFactories, fakeClockTransportTracer); } @@ -67,7 +67,7 @@ protected List newServer( return AccessProtectedHack.serverBuilderBuildTransportServer( NettyServerBuilder .forAddress(new InetSocketAddress(port)) - .flowControlWindow(65 * 1024), + .flowControlWindow(AbstractTransportTest.TEST_FLOW_CONTROL_WINDOW), streamTracerFactories, fakeClockTransportTracer); } From 522e70d1d7c70c50d5a21ad4578271cafe3aefd7 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Wed, 2 Sep 2020 19:03:15 -0400 Subject: [PATCH 16/86] api: remove incomplete sentence in javadoc --- api/src/main/java/io/grpc/ServerBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/io/grpc/ServerBuilder.java b/api/src/main/java/io/grpc/ServerBuilder.java index 364885e026f..64a63ad2acc 100644 --- a/api/src/main/java/io/grpc/ServerBuilder.java +++ b/api/src/main/java/io/grpc/ServerBuilder.java @@ -118,7 +118,7 @@ public T addTransportFilter(ServerTransportFilter filter) { /** * Adds a {@link ServerStreamTracer.Factory} to measure server-side traffic. The order of - * factories being added is the order they will be executed. Tracers should not + * factories being added is the order they will be executed. * * @return this * @since 1.3.0 From 07b812b1f52767e9a25b21148dee96b1c990bf87 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Mon, 31 Aug 2020 18:59:52 -0400 Subject: [PATCH 17/86] api, core: create ForwardingServerBuilder and ServerImplBuilder --- .../java/io/grpc/ForwardingServerBuilder.java | 169 ++++++++++++++++++ .../io/grpc/ForwardingServerBuilderTest.java | 83 +++++++++ .../grpc/benchmarks/TransportBenchmark.java | 4 +- .../internal/AbstractServerImplBuilder.java | 2 +- .../io/grpc/internal/ServerImplBuilder.java | 105 +++++++++++ .../grpc/internal/ServerImplBuilderTest.java | 64 +++++++ 6 files changed, 424 insertions(+), 3 deletions(-) create mode 100644 api/src/main/java/io/grpc/ForwardingServerBuilder.java create mode 100644 api/src/test/java/io/grpc/ForwardingServerBuilderTest.java create mode 100644 core/src/main/java/io/grpc/internal/ServerImplBuilder.java create mode 100644 core/src/test/java/io/grpc/internal/ServerImplBuilderTest.java diff --git a/api/src/main/java/io/grpc/ForwardingServerBuilder.java b/api/src/main/java/io/grpc/ForwardingServerBuilder.java new file mode 100644 index 00000000000..4e1d5a1baf5 --- /dev/null +++ b/api/src/main/java/io/grpc/ForwardingServerBuilder.java @@ -0,0 +1,169 @@ +/* + * Copyright 2020 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc; + +import com.google.common.base.MoreObjects; +import java.io.File; +import java.io.InputStream; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; + +/** + * A {@link ServerBuilder} that delegates all its builder method to another builder by default. + * + * @param The type of the subclass extending this abstract class. + * @since 1.33.0 + */ +@ExperimentalApi("https://github.com/grpc/grpc-java/issues/7393") +public abstract class ForwardingServerBuilder> + extends ServerBuilder { + + /** The default constructor. */ + protected ForwardingServerBuilder() {} + + /** + * This method serves to force sub classes to "hide" this static factory. + */ + public static ServerBuilder forPort(int port) { + throw new UnsupportedOperationException("Subclass failed to hide static factory"); + } + + /** + * Returns the delegated {@code ServerBuilder}. + */ + protected abstract ServerBuilder delegate(); + + @Override + public T directExecutor() { + delegate().directExecutor(); + return thisT(); + } + + @Override + public T executor(@Nullable Executor executor) { + delegate().executor(executor); + return thisT(); + } + + @Override + public T addService(ServerServiceDefinition service) { + delegate().addService(service); + return thisT(); + } + + @Override + public T addService(BindableService bindableService) { + delegate().addService(bindableService); + return thisT(); + } + + @Override + public T intercept(ServerInterceptor interceptor) { + delegate().intercept(interceptor); + return thisT(); + } + + @Override + public T addTransportFilter(ServerTransportFilter filter) { + delegate().addTransportFilter(filter); + return thisT(); + } + + @Override + public T addStreamTracerFactory(ServerStreamTracer.Factory factory) { + delegate().addStreamTracerFactory(factory); + return thisT(); + } + + @Override + public T fallbackHandlerRegistry(@Nullable HandlerRegistry fallbackRegistry) { + delegate().fallbackHandlerRegistry(fallbackRegistry); + return thisT(); + } + + @Override + public T useTransportSecurity(File certChain, File privateKey) { + delegate().useTransportSecurity(certChain, privateKey); + return thisT(); + } + + @Override + public T useTransportSecurity(InputStream certChain, InputStream privateKey) { + delegate().useTransportSecurity(certChain, privateKey); + return thisT(); + } + + @Override + public T decompressorRegistry(@Nullable DecompressorRegistry registry) { + delegate().decompressorRegistry(registry); + return thisT(); + } + + @Override + public T compressorRegistry(@Nullable CompressorRegistry registry) { + delegate().compressorRegistry(registry); + return thisT(); + } + + @Override + public T handshakeTimeout(long timeout, TimeUnit unit) { + delegate().handshakeTimeout(timeout, unit); + return thisT(); + } + + @Override + public T maxInboundMessageSize(int bytes) { + delegate().maxInboundMessageSize(bytes); + return thisT(); + } + + @Override + public T maxInboundMetadataSize(int bytes) { + delegate().maxInboundMetadataSize(bytes); + return thisT(); + } + + @Override + public T setBinaryLog(BinaryLog binaryLog) { + delegate().setBinaryLog(binaryLog); + return thisT(); + } + + /** + * Returns the {@link Server} built by the delegate by default. Overriding method can return + * different value. + */ + @Override + public Server build() { + return delegate().build(); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("delegate", delegate()).toString(); + } + + /** + * Returns the correctly typed version of the builder. + */ + protected final T thisT() { + @SuppressWarnings("unchecked") + T thisT = (T) this; + return thisT; + } +} diff --git a/api/src/test/java/io/grpc/ForwardingServerBuilderTest.java b/api/src/test/java/io/grpc/ForwardingServerBuilderTest.java new file mode 100644 index 00000000000..6a8c1c115a9 --- /dev/null +++ b/api/src/test/java/io/grpc/ForwardingServerBuilderTest.java @@ -0,0 +1,83 @@ +/* + * Copyright 2020 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; + +import com.google.common.base.Defaults; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.Collections; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Unit tests for {@link ForwardingServerBuilder}. + */ +@RunWith(JUnit4.class) +public class ForwardingServerBuilderTest { + private final ServerBuilder mockDelegate = mock(ServerBuilder.class); + private final ForwardingServerBuilder testServerBuilder = new TestBuilder(); + + private final class TestBuilder extends ForwardingServerBuilder { + @Override + protected ServerBuilder delegate() { + return mockDelegate; + } + } + + @Test + public void allMethodsForwarded() throws Exception { + ForwardingTestUtil.testMethodsForwarded( + ServerBuilder.class, + mockDelegate, + testServerBuilder, + Collections.emptyList()); + } + + @Test + public void allBuilderMethodsReturnThis() throws Exception { + for (Method method : ServerBuilder.class.getDeclaredMethods()) { + if (Modifier.isStatic(method.getModifiers()) || Modifier.isPrivate(method.getModifiers())) { + continue; + } + if (method.getName().equals("build")) { + continue; + } + Class[] argTypes = method.getParameterTypes(); + Object[] args = new Object[argTypes.length]; + for (int i = 0; i < argTypes.length; i++) { + args[i] = Defaults.defaultValue(argTypes[i]); + } + + Object returnedValue = method.invoke(testServerBuilder, args); + + assertThat(returnedValue).isSameInstanceAs(testServerBuilder); + } + } + + @Test + public void buildReturnsDelegateBuildByDefault() { + Server server = mock(Server.class); + doReturn(server).when(mockDelegate).build(); + + assertThat(testServerBuilder.build()).isSameInstanceAs(server); + } +} diff --git a/benchmarks/src/jmh/java/io/grpc/benchmarks/TransportBenchmark.java b/benchmarks/src/jmh/java/io/grpc/benchmarks/TransportBenchmark.java index ec4646bff44..b673657cb35 100644 --- a/benchmarks/src/jmh/java/io/grpc/benchmarks/TransportBenchmark.java +++ b/benchmarks/src/jmh/java/io/grpc/benchmarks/TransportBenchmark.java @@ -22,6 +22,7 @@ import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.grpc.Server; +import io.grpc.ServerBuilder; import io.grpc.Status; import io.grpc.StatusRuntimeException; import io.grpc.benchmarks.proto.BenchmarkServiceGrpc; @@ -31,7 +32,6 @@ import io.grpc.benchmarks.qps.AsyncServer; import io.grpc.inprocess.InProcessChannelBuilder; import io.grpc.inprocess.InProcessServerBuilder; -import io.grpc.internal.AbstractServerImplBuilder; import io.grpc.netty.NegotiationType; import io.grpc.netty.NettyChannelBuilder; import io.grpc.netty.NettyServerBuilder; @@ -80,7 +80,7 @@ public enum Transport { @Setup public void setUp() throws Exception { - AbstractServerImplBuilder serverBuilder; + ServerBuilder serverBuilder; ManagedChannelBuilder channelBuilder; switch (transport) { case INPROCESS: diff --git a/core/src/main/java/io/grpc/internal/AbstractServerImplBuilder.java b/core/src/main/java/io/grpc/internal/AbstractServerImplBuilder.java index 21a1becc284..c68076cc4c6 100644 --- a/core/src/main/java/io/grpc/internal/AbstractServerImplBuilder.java +++ b/core/src/main/java/io/grpc/internal/AbstractServerImplBuilder.java @@ -283,7 +283,7 @@ final List getTracerFactories() { return Collections.unmodifiableList(tracerFactories); } - protected final InternalChannelz getChannelz() { + protected InternalChannelz getChannelz() { return channelz; } diff --git a/core/src/main/java/io/grpc/internal/ServerImplBuilder.java b/core/src/main/java/io/grpc/internal/ServerImplBuilder.java new file mode 100644 index 00000000000..ce92f1c65c3 --- /dev/null +++ b/core/src/main/java/io/grpc/internal/ServerImplBuilder.java @@ -0,0 +1,105 @@ +/* + * Copyright 2020 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.internal; + +import com.google.common.base.Preconditions; +import io.grpc.Deadline; +import io.grpc.InternalChannelz; +import io.grpc.ServerBuilder; +import io.grpc.ServerStreamTracer; +import java.io.File; +import java.util.List; +import java.util.concurrent.Executor; + +/** + * Default builder for {@link io.grpc.Server} instances, for usage in Transport implementations. + */ +public final class ServerImplBuilder extends AbstractServerImplBuilder { + private final ClientTransportServersBuilder clientTransportServersBuilder; + + /** + * An interface to provide to provide transport specific information for the server. This method + * is meant for Transport implementors and should not be used by normal users. + */ + public interface ClientTransportServersBuilder { + List buildClientTransportServers( + List streamTracerFactories); + } + + /** + * Creates a new server builder with given transport servers provider. + */ + public ServerImplBuilder(ClientTransportServersBuilder clientTransportServersBuilder) { + this.clientTransportServersBuilder = Preconditions + .checkNotNull(clientTransportServersBuilder, "clientTransportServersBuilder"); + } + + @Override + protected List buildTransportServers( + List streamTracerFactories) { + return clientTransportServersBuilder.buildClientTransportServers(streamTracerFactories); + } + + @Override + public void setDeadlineTicker(Deadline.Ticker ticker) { + super.setDeadlineTicker(ticker); + } + + @Override + public void setTracingEnabled(boolean value) { + super.setTracingEnabled(value); + } + + @Override + public void setStatsEnabled(boolean value) { + super.setStatsEnabled(value); + } + + @Override + public void setStatsRecordStartedRpcs(boolean value) { + super.setStatsRecordStartedRpcs(value); + } + + @Override + public void setStatsRecordFinishedRpcs(boolean value) { + super.setStatsRecordFinishedRpcs(value); + } + + @Override + public void setStatsRecordRealTimeMetrics(boolean value) { + super.setStatsRecordRealTimeMetrics(value); + } + + @Override + public InternalChannelz getChannelz() { + return super.getChannelz(); + } + + @Override + public ObjectPool getExecutorPool() { + return super.getExecutorPool(); + } + + @Override + public ServerImplBuilder useTransportSecurity(File certChain, File privateKey) { + throw new UnsupportedOperationException("TLS not supported in ServerImplBuilder"); + } + + public static ServerBuilder forPort(int port) { + throw new UnsupportedOperationException("ClientTransportServersBuilder is required"); + } +} diff --git a/core/src/test/java/io/grpc/internal/ServerImplBuilderTest.java b/core/src/test/java/io/grpc/internal/ServerImplBuilderTest.java new file mode 100644 index 00000000000..3b5e3c7d8c3 --- /dev/null +++ b/core/src/test/java/io/grpc/internal/ServerImplBuilderTest.java @@ -0,0 +1,64 @@ +/* + * Copyright 2020 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.internal; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.verify; + +import io.grpc.ServerStreamTracer; +import io.grpc.internal.ServerImplBuilder.ClientTransportServersBuilder; +import java.util.List; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.ArgumentMatchers; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** Unit tests for {@link ServerImplBuilder}. */ +@RunWith(JUnit4.class) +public class ServerImplBuilderTest { + @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + + @Mock private ClientTransportServersBuilder mockClientTransportServersBuilder; + @Mock private List mockServerStreamTracerFactories; + @Mock private List mockInternalServers; + private ServerImplBuilder builder; + + @Before + public void setUp() throws Exception { + builder = new ServerImplBuilder(mockClientTransportServersBuilder); + } + + @Test + public void buildTransportServers() { + doReturn(mockInternalServers).when(mockClientTransportServersBuilder) + .buildClientTransportServers(ArgumentMatchers.anyList()); + + List servers = builder + .buildTransportServers(mockServerStreamTracerFactories); + assertEquals(mockInternalServers, servers); + assertNotNull(servers); + verify(mockClientTransportServersBuilder) + .buildClientTransportServers(mockServerStreamTracerFactories); + } +} From b03f148ed9d2c91815442daa5b2a0f0003ed0cab Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Wed, 2 Sep 2020 16:47:15 -0400 Subject: [PATCH 18/86] core, netty: server builders extend a public API class --- .../inprocess/InProcessServerBuilder.java | 41 +++++++++--- .../InternalInProcessServerBuilder.java | 32 ++++++++++ .../integration/AbstractInteropTest.java | 34 +++++----- .../integration/AutoWindowSizingOnTest.java | 10 ++- .../Http2NettyLocalChannelTest.java | 10 ++- .../testing/integration/Http2NettyTest.java | 10 ++- .../testing/integration/Http2OkHttpTest.java | 10 ++- .../testing/integration/InProcessTest.java | 16 ++++- .../integration/TransportCompressionTest.java | 10 ++- .../netty/InternalNettyServerBuilder.java | 14 ++++- .../io/grpc/netty/NettyServerBuilder.java | 62 +++++++++++++------ .../io/grpc/okhttp/OkHttpTransportTest.java | 24 ++++--- 12 files changed, 196 insertions(+), 77 deletions(-) create mode 100644 core/src/main/java/io/grpc/inprocess/InternalInProcessServerBuilder.java diff --git a/core/src/main/java/io/grpc/inprocess/InProcessServerBuilder.java b/core/src/main/java/io/grpc/inprocess/InProcessServerBuilder.java index 603a794e3db..37e6dcaede5 100644 --- a/core/src/main/java/io/grpc/inprocess/InProcessServerBuilder.java +++ b/core/src/main/java/io/grpc/inprocess/InProcessServerBuilder.java @@ -21,11 +21,16 @@ import com.google.common.base.Preconditions; import io.grpc.Deadline; import io.grpc.ExperimentalApi; +import io.grpc.ForwardingServerBuilder; +import io.grpc.Internal; +import io.grpc.ServerBuilder; import io.grpc.ServerStreamTracer; -import io.grpc.internal.AbstractServerImplBuilder; import io.grpc.internal.FixedObjectPool; import io.grpc.internal.GrpcUtil; +import io.grpc.internal.InternalServer; import io.grpc.internal.ObjectPool; +import io.grpc.internal.ServerImplBuilder; +import io.grpc.internal.ServerImplBuilder.ClientTransportServersBuilder; import io.grpc.internal.SharedResourcePool; import java.io.File; import java.util.Collections; @@ -67,8 +72,7 @@ * */ @ExperimentalApi("https://github.com/grpc/grpc-java/issues/1783") -public final class InProcessServerBuilder - extends AbstractServerImplBuilder { +public final class InProcessServerBuilder extends ForwardingServerBuilder { /** * Create a server builder that will bind with the given name. * @@ -93,6 +97,7 @@ public static String generateName() { return UUID.randomUUID().toString(); } + private final ServerImplBuilder serverImplBuilder; final String name; int maxInboundMetadataSize = Integer.MAX_VALUE; ObjectPool schedulerPool = @@ -100,15 +105,32 @@ public static String generateName() { private InProcessServerBuilder(String name) { this.name = Preconditions.checkNotNull(name, "name"); + + final class InProcessClientTransportServersBuilder implements ClientTransportServersBuilder { + @Override + public List buildClientTransportServers( + List streamTracerFactories) { + return buildTransportServers(streamTracerFactories); + } + } + + serverImplBuilder = new ServerImplBuilder(new InProcessClientTransportServersBuilder()); + // In-process transport should not record its traffic to the stats module. // https://github.com/grpc/grpc-java/issues/2284 - setStatsRecordStartedRpcs(false); - setStatsRecordFinishedRpcs(false); + serverImplBuilder.setStatsRecordStartedRpcs(false); + serverImplBuilder.setStatsRecordFinishedRpcs(false); // Disable handshake timeout because it is unnecessary, and can trigger Thread creation that can // break some environments (like tests). handshakeTimeout(Long.MAX_VALUE, TimeUnit.SECONDS); } + @Internal + @Override + protected ServerBuilder delegate() { + return serverImplBuilder; + } + /** * Provides a custom scheduled executor service. * @@ -140,7 +162,7 @@ public InProcessServerBuilder scheduledExecutorService( * @since 1.24.0 */ public InProcessServerBuilder deadlineTicker(Deadline.Ticker ticker) { - setDeadlineTicker(ticker); + serverImplBuilder.setDeadlineTicker(ticker); return this; } @@ -164,8 +186,7 @@ public InProcessServerBuilder maxInboundMetadataSize(int bytes) { return this; } - @Override - protected List buildTransportServers( + List buildTransportServers( List streamTracerFactories) { return Collections.singletonList(new InProcessServer(this, streamTracerFactories)); } @@ -174,4 +195,8 @@ protected List buildTransportServers( public InProcessServerBuilder useTransportSecurity(File certChain, File privateKey) { throw new UnsupportedOperationException("TLS not supported in InProcessServer"); } + + void setStatsEnabled(boolean value) { + this.serverImplBuilder.setStatsEnabled(value); + } } diff --git a/core/src/main/java/io/grpc/inprocess/InternalInProcessServerBuilder.java b/core/src/main/java/io/grpc/inprocess/InternalInProcessServerBuilder.java new file mode 100644 index 00000000000..df531707451 --- /dev/null +++ b/core/src/main/java/io/grpc/inprocess/InternalInProcessServerBuilder.java @@ -0,0 +1,32 @@ +/* + * Copyright 2020 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.inprocess; + +import io.grpc.Internal; + +/** + * Internal {@link InProcessServerBuilder} accessor. This is intended for usage internal to + * the gRPC team. If you *really* think you need to use this, contact the gRPC team first. + */ +@Internal +public class InternalInProcessServerBuilder { + public static void setStatsEnabled(InProcessServerBuilder builder, boolean value) { + builder.setStatsEnabled(value); + } + + private InternalInProcessServerBuilder() {} +} diff --git a/interop-testing/src/main/java/io/grpc/testing/integration/AbstractInteropTest.java b/interop-testing/src/main/java/io/grpc/testing/integration/AbstractInteropTest.java index 643d984bad8..769309a5a1a 100644 --- a/interop-testing/src/main/java/io/grpc/testing/integration/AbstractInteropTest.java +++ b/interop-testing/src/main/java/io/grpc/testing/integration/AbstractInteropTest.java @@ -64,7 +64,6 @@ import io.grpc.auth.MoreCallCredentials; import io.grpc.census.InternalCensusStatsAccessor; import io.grpc.census.internal.DeprecatedCensusConstants; -import io.grpc.internal.AbstractServerImplBuilder; import io.grpc.internal.GrpcUtil; import io.grpc.internal.testing.StatsTestUtils; import io.grpc.internal.testing.StatsTestUtils.FakeStatsRecorder; @@ -171,7 +170,6 @@ public abstract class AbstractInteropTest { private ScheduledExecutorService testServiceExecutor; private Server server; - private boolean customCensusModulePresent; private final LinkedBlockingQueue serverStreamTracers = new LinkedBlockingQueue<>(); @@ -245,21 +243,7 @@ private void startServer() { new TestServiceImpl(testServiceExecutor), allInterceptors)) .addStreamTracerFactory(serverStreamTracerFactory); - if (builder instanceof AbstractServerImplBuilder) { - customCensusModulePresent = true; - ServerStreamTracer.Factory censusTracerFactory = - InternalCensusStatsAccessor - .getServerStreamTracerFactory( - tagger, tagContextBinarySerializer, serverStatsRecorder, - GrpcUtil.STOPWATCH_SUPPLIER, - true, true, true, false /* real-time metrics */); - AbstractServerImplBuilder sb = (AbstractServerImplBuilder) builder; - io.grpc.internal.TestingAccessor.setStatsEnabled(sb, false); - sb.addStreamTracerFactory(censusTracerFactory); - } - if (metricsExpected()) { - assertThat(builder).isInstanceOf(AbstractServerImplBuilder.class); - } + try { server = builder.build().start(); } catch (IOException ex) { @@ -373,6 +357,20 @@ protected final ClientInterceptor createCensusStatsClientInterceptor() { true, true, true, false /* real-time metrics */); } + protected final ServerStreamTracer.Factory createCustomCensusTracerFactory() { + return InternalCensusStatsAccessor.getServerStreamTracerFactory( + tagger, tagContextBinarySerializer, serverStatsRecorder, + GrpcUtil.STOPWATCH_SUPPLIER, + true, true, true, false /* real-time metrics */); + } + + /** + * Override this when custom census module presence is different from {@link #metricsExpected()}. + */ + protected boolean customCensusModulePresent() { + return metricsExpected(); + } + /** * Return true if exact metric values should be checked. */ @@ -1510,7 +1508,7 @@ public void customMetadata() throws Exception { @Test(timeout = 10000) public void censusContextsPropagated() { Assume.assumeTrue("Skip the test because server is not in the same process.", server != null); - Assume.assumeTrue(customCensusModulePresent); + Assume.assumeTrue(customCensusModulePresent()); Span clientParentSpan = Tracing.getTracer().spanBuilder("Test.interopTest").startSpan(); // A valid ID is guaranteed to be unique, so we can verify it is actually propagated. assertTrue(clientParentSpan.getContext().getTraceId().isValid()); diff --git a/interop-testing/src/test/java/io/grpc/testing/integration/AutoWindowSizingOnTest.java b/interop-testing/src/test/java/io/grpc/testing/integration/AutoWindowSizingOnTest.java index a2036ecea91..05fd970e3e0 100644 --- a/interop-testing/src/test/java/io/grpc/testing/integration/AutoWindowSizingOnTest.java +++ b/interop-testing/src/test/java/io/grpc/testing/integration/AutoWindowSizingOnTest.java @@ -16,8 +16,9 @@ package io.grpc.testing.integration; -import io.grpc.internal.AbstractServerImplBuilder; +import io.grpc.ServerBuilder; import io.grpc.netty.InternalNettyChannelBuilder; +import io.grpc.netty.InternalNettyServerBuilder; import io.grpc.netty.NegotiationType; import io.grpc.netty.NettyChannelBuilder; import io.grpc.netty.NettyServerBuilder; @@ -28,9 +29,12 @@ public class AutoWindowSizingOnTest extends AbstractInteropTest { @Override - protected AbstractServerImplBuilder getServerBuilder() { - return NettyServerBuilder.forPort(0) + protected ServerBuilder getServerBuilder() { + NettyServerBuilder builder = NettyServerBuilder.forPort(0) .maxInboundMessageSize(AbstractInteropTest.MAX_MESSAGE_SIZE); + // Disable the default census stats tracer, use testing tracer instead. + InternalNettyServerBuilder.setStatsEnabled(builder, false); + return builder.addStreamTracerFactory(createCustomCensusTracerFactory()); } @Override diff --git a/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyLocalChannelTest.java b/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyLocalChannelTest.java index 4494d620b1e..778e8bdf97c 100644 --- a/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyLocalChannelTest.java +++ b/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyLocalChannelTest.java @@ -16,8 +16,9 @@ package io.grpc.testing.integration; -import io.grpc.internal.AbstractServerImplBuilder; +import io.grpc.ServerBuilder; import io.grpc.netty.InternalNettyChannelBuilder; +import io.grpc.netty.InternalNettyServerBuilder; import io.grpc.netty.NegotiationType; import io.grpc.netty.NettyChannelBuilder; import io.grpc.netty.NettyServerBuilder; @@ -38,14 +39,17 @@ public class Http2NettyLocalChannelTest extends AbstractInteropTest { private DefaultEventLoopGroup eventLoopGroup = new DefaultEventLoopGroup(); @Override - protected AbstractServerImplBuilder getServerBuilder() { - return NettyServerBuilder + protected ServerBuilder getServerBuilder() { + NettyServerBuilder builder = NettyServerBuilder .forAddress(new LocalAddress("in-process-1")) .flowControlWindow(AbstractInteropTest.TEST_FLOW_CONTROL_WINDOW) .maxInboundMessageSize(AbstractInteropTest.MAX_MESSAGE_SIZE) .channelType(LocalServerChannel.class) .workerEventLoopGroup(eventLoopGroup) .bossEventLoopGroup(eventLoopGroup); + // Disable the default census stats tracer, use testing tracer instead. + InternalNettyServerBuilder.setStatsEnabled(builder, false); + return builder.addStreamTracerFactory(createCustomCensusTracerFactory()); } @Override diff --git a/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyTest.java b/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyTest.java index 9053a1fff64..0d391039105 100644 --- a/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyTest.java +++ b/interop-testing/src/test/java/io/grpc/testing/integration/Http2NettyTest.java @@ -19,10 +19,11 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; -import io.grpc.internal.AbstractServerImplBuilder; +import io.grpc.ServerBuilder; import io.grpc.internal.testing.TestUtils; import io.grpc.netty.GrpcSslContexts; import io.grpc.netty.InternalNettyChannelBuilder; +import io.grpc.netty.InternalNettyServerBuilder; import io.grpc.netty.NettyChannelBuilder; import io.grpc.netty.NettyServerBuilder; import io.netty.handler.ssl.ClientAuth; @@ -41,10 +42,10 @@ public class Http2NettyTest extends AbstractInteropTest { @Override - protected AbstractServerImplBuilder getServerBuilder() { + protected ServerBuilder getServerBuilder() { // Starts the server with HTTPS. try { - return NettyServerBuilder.forPort(0) + NettyServerBuilder builder = NettyServerBuilder.forPort(0) .flowControlWindow(AbstractInteropTest.TEST_FLOW_CONTROL_WINDOW) .maxInboundMessageSize(AbstractInteropTest.MAX_MESSAGE_SIZE) .sslContext(GrpcSslContexts @@ -53,6 +54,9 @@ protected AbstractServerImplBuilder getServerBuilder() { .trustManager(TestUtils.loadCert("ca.pem")) .ciphers(TestUtils.preferredTestCiphers(), SupportedCipherSuiteFilter.INSTANCE) .build()); + // Disable the default census stats tracer, use testing tracer instead. + InternalNettyServerBuilder.setStatsEnabled(builder, false); + return builder.addStreamTracerFactory(createCustomCensusTracerFactory()); } catch (IOException ex) { throw new RuntimeException(ex); } diff --git a/interop-testing/src/test/java/io/grpc/testing/integration/Http2OkHttpTest.java b/interop-testing/src/test/java/io/grpc/testing/integration/Http2OkHttpTest.java index ea4ba5877ce..612774c7564 100644 --- a/interop-testing/src/test/java/io/grpc/testing/integration/Http2OkHttpTest.java +++ b/interop-testing/src/test/java/io/grpc/testing/integration/Http2OkHttpTest.java @@ -23,11 +23,12 @@ import com.google.common.base.Throwables; import com.squareup.okhttp.ConnectionSpec; import io.grpc.ManagedChannel; -import io.grpc.internal.AbstractServerImplBuilder; +import io.grpc.ServerBuilder; import io.grpc.internal.GrpcUtil; import io.grpc.internal.testing.StreamRecorder; import io.grpc.internal.testing.TestUtils; import io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.InternalNettyServerBuilder; import io.grpc.netty.NettyServerBuilder; import io.grpc.okhttp.InternalOkHttpChannelBuilder; import io.grpc.okhttp.OkHttpChannelBuilder; @@ -64,7 +65,7 @@ public static void loadConscrypt() throws Exception { } @Override - protected AbstractServerImplBuilder getServerBuilder() { + protected ServerBuilder getServerBuilder() { // Starts the server with HTTPS. try { SslProvider sslProvider = SslContext.defaultServerProvider(); @@ -77,10 +78,13 @@ protected AbstractServerImplBuilder getServerBuilder() { .forServer(TestUtils.loadCert("server1.pem"), TestUtils.loadCert("server1.key")); GrpcSslContexts.configure(contextBuilder, sslProvider); contextBuilder.ciphers(TestUtils.preferredTestCiphers(), SupportedCipherSuiteFilter.INSTANCE); - return NettyServerBuilder.forPort(0) + NettyServerBuilder builder = NettyServerBuilder.forPort(0) .flowControlWindow(AbstractInteropTest.TEST_FLOW_CONTROL_WINDOW) .maxInboundMessageSize(AbstractInteropTest.MAX_MESSAGE_SIZE) .sslContext(contextBuilder.build()); + // Disable the default census stats tracer, use testing tracer instead. + InternalNettyServerBuilder.setStatsEnabled(builder, false); + return builder.addStreamTracerFactory(createCustomCensusTracerFactory()); } catch (IOException ex) { throw new RuntimeException(ex); } diff --git a/interop-testing/src/test/java/io/grpc/testing/integration/InProcessTest.java b/interop-testing/src/test/java/io/grpc/testing/integration/InProcessTest.java index 66894834b95..1ad63ae85d8 100644 --- a/interop-testing/src/test/java/io/grpc/testing/integration/InProcessTest.java +++ b/interop-testing/src/test/java/io/grpc/testing/integration/InProcessTest.java @@ -16,10 +16,11 @@ package io.grpc.testing.integration; +import io.grpc.ServerBuilder; import io.grpc.inprocess.InProcessChannelBuilder; import io.grpc.inprocess.InProcessServerBuilder; import io.grpc.inprocess.InternalInProcessChannelBuilder; -import io.grpc.internal.AbstractServerImplBuilder; +import io.grpc.inprocess.InternalInProcessServerBuilder; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -30,9 +31,12 @@ public class InProcessTest extends AbstractInteropTest { private static final String SERVER_NAME = "test"; @Override - protected AbstractServerImplBuilder getServerBuilder() { + protected ServerBuilder getServerBuilder() { // Starts the in-process server. - return InProcessServerBuilder.forName(SERVER_NAME); + InProcessServerBuilder builder = InProcessServerBuilder.forName(SERVER_NAME); + // Disable the default census stats tracer, use testing tracer instead. + InternalInProcessServerBuilder.setStatsEnabled(builder, false); + return builder.addStreamTracerFactory(createCustomCensusTracerFactory()); } @Override @@ -43,6 +47,12 @@ protected InProcessChannelBuilder createChannelBuilder() { return builder.intercept(createCensusStatsClientInterceptor()); } + @Override + protected boolean customCensusModulePresent() { + // Metrics values are not expected, but custom census module is still used. + return true; + } + @Override protected boolean metricsExpected() { // TODO(zhangkun83): InProcessTransport by-passes framer and deframer, thus message sizes are diff --git a/interop-testing/src/test/java/io/grpc/testing/integration/TransportCompressionTest.java b/interop-testing/src/test/java/io/grpc/testing/integration/TransportCompressionTest.java index 1144c75073c..8d184dd4428 100644 --- a/interop-testing/src/test/java/io/grpc/testing/integration/TransportCompressionTest.java +++ b/interop-testing/src/test/java/io/grpc/testing/integration/TransportCompressionTest.java @@ -31,13 +31,14 @@ import io.grpc.ForwardingClientCallListener; import io.grpc.Metadata; import io.grpc.MethodDescriptor; +import io.grpc.ServerBuilder; import io.grpc.ServerCall; import io.grpc.ServerCall.Listener; import io.grpc.ServerCallHandler; import io.grpc.ServerInterceptor; -import io.grpc.internal.AbstractServerImplBuilder; import io.grpc.internal.GrpcUtil; import io.grpc.netty.InternalNettyChannelBuilder; +import io.grpc.netty.InternalNettyServerBuilder; import io.grpc.netty.NettyChannelBuilder; import io.grpc.netty.NettyServerBuilder; import io.grpc.testing.integration.Messages.BoolValue; @@ -83,8 +84,8 @@ public static void registerCompressors() { } @Override - protected AbstractServerImplBuilder getServerBuilder() { - return NettyServerBuilder.forPort(0) + protected ServerBuilder getServerBuilder() { + NettyServerBuilder builder = NettyServerBuilder.forPort(0) .maxInboundMessageSize(AbstractInteropTest.MAX_MESSAGE_SIZE) .compressorRegistry(compressors) .decompressorRegistry(decompressors) @@ -98,6 +99,9 @@ public Listener interceptCall(ServerCall call, return listener; } }); + // Disable the default census stats tracer, use testing tracer instead. + InternalNettyServerBuilder.setStatsEnabled(builder, false); + return builder.addStreamTracerFactory(createCustomCensusTracerFactory()); } @Test diff --git a/netty/src/main/java/io/grpc/netty/InternalNettyServerBuilder.java b/netty/src/main/java/io/grpc/netty/InternalNettyServerBuilder.java index e89593364b3..d878e11d4d0 100644 --- a/netty/src/main/java/io/grpc/netty/InternalNettyServerBuilder.java +++ b/netty/src/main/java/io/grpc/netty/InternalNettyServerBuilder.java @@ -17,15 +17,27 @@ package io.grpc.netty; import io.grpc.Internal; +import io.grpc.ServerStreamTracer; import io.grpc.internal.SharedResourcePool; +import io.grpc.internal.TransportTracer; import io.netty.channel.socket.nio.NioServerSocketChannel; +import java.util.List; /** - * Internal {@link InternalNettyServerBuilder} accessor. This is intended for usage internal to + * Internal {@link NettyServerBuilder} accessor. This is intended for usage internal to * the gRPC team. If you *really* think you need to use this, contact the gRPC team first. */ @Internal public final class InternalNettyServerBuilder { + public static List buildTransportServers(NettyServerBuilder builder, + List streamTracerFactories) { + return builder.buildTransportServers(streamTracerFactories); + } + + public static void setTransportTracerFactory(NettyServerBuilder builder, + TransportTracer.Factory transportTracerFactory) { + builder.setTransportTracerFactory(transportTracerFactory); + } public static void setStatsEnabled(NettyServerBuilder builder, boolean value) { builder.setStatsEnabled(value); diff --git a/netty/src/main/java/io/grpc/netty/NettyServerBuilder.java b/netty/src/main/java/io/grpc/netty/NettyServerBuilder.java index 4560d697da6..8bda3d7c91e 100644 --- a/netty/src/main/java/io/grpc/netty/NettyServerBuilder.java +++ b/netty/src/main/java/io/grpc/netty/NettyServerBuilder.java @@ -27,14 +27,19 @@ import com.google.common.annotations.VisibleForTesting; import com.google.errorprone.annotations.CanIgnoreReturnValue; import io.grpc.ExperimentalApi; +import io.grpc.ForwardingServerBuilder; import io.grpc.Internal; +import io.grpc.ServerBuilder; import io.grpc.ServerStreamTracer; -import io.grpc.internal.AbstractServerImplBuilder; import io.grpc.internal.FixedObjectPool; import io.grpc.internal.GrpcUtil; +import io.grpc.internal.InternalServer; import io.grpc.internal.KeepAliveManager; import io.grpc.internal.ObjectPool; +import io.grpc.internal.ServerImplBuilder; +import io.grpc.internal.ServerImplBuilder.ClientTransportServersBuilder; import io.grpc.internal.SharedResourcePool; +import io.grpc.internal.TransportTracer; import io.netty.channel.ChannelFactory; import io.netty.channel.ChannelOption; import io.netty.channel.EventLoopGroup; @@ -61,7 +66,7 @@ */ @ExperimentalApi("https://github.com/grpc/grpc-java/issues/1784") @CanIgnoreReturnValue -public final class NettyServerBuilder extends AbstractServerImplBuilder { +public final class NettyServerBuilder extends ForwardingServerBuilder { // 1MiB public static final int DEFAULT_FLOW_CONTROL_WINDOW = 1024 * 1024; @@ -80,8 +85,10 @@ public final class NettyServerBuilder extends AbstractServerImplBuilder DEFAULT_WORKER_EVENT_LOOP_GROUP_POOL = SharedResourcePool.forResource(Utils.DEFAULT_WORKER_EVENT_LOOP_GROUP); + private final ServerImplBuilder serverImplBuilder; private final List listenAddresses = new ArrayList<>(); + private TransportTracer.Factory transportTracerFactory = TransportTracer.getDefaultFactory(); private ChannelFactory channelFactory = Utils.DEFAULT_SERVER_CHANNEL_FACTORY; private final Map, Object> channelOptions = new HashMap<>(); @@ -128,16 +135,32 @@ public static NettyServerBuilder forAddress(SocketAddress address) { return new NettyServerBuilder(address); } + private final class NettyClientTransportServersBuilder implements ClientTransportServersBuilder { + @Override + public List buildClientTransportServers( + List streamTracerFactories) { + return buildTransportServers(streamTracerFactories); + } + } + @CheckReturnValue private NettyServerBuilder(int port) { + serverImplBuilder = new ServerImplBuilder(new NettyClientTransportServersBuilder()); this.listenAddresses.add(new InetSocketAddress(port)); } @CheckReturnValue private NettyServerBuilder(SocketAddress address) { + serverImplBuilder = new ServerImplBuilder(new NettyClientTransportServersBuilder()); this.listenAddresses.add(address); } + @Internal + @Override + protected ServerBuilder delegate() { + return serverImplBuilder; + } + /** * Adds an additional address for this server to listen on. Callers must ensure that all socket * addresses are compatible with the Netty channel type, and that they don't conflict with each @@ -316,24 +339,20 @@ public final NettyServerBuilder protocolNegotiator( return this; } - @Override - protected void setTracingEnabled(boolean value) { - super.setTracingEnabled(value); + void setTracingEnabled(boolean value) { + this.serverImplBuilder.setTracingEnabled(value); } - @Override - protected void setStatsEnabled(boolean value) { - super.setStatsEnabled(value); + void setStatsEnabled(boolean value) { + this.serverImplBuilder.setStatsEnabled(value); } - @Override - protected void setStatsRecordStartedRpcs(boolean value) { - super.setStatsRecordStartedRpcs(value); + void setStatsRecordStartedRpcs(boolean value) { + this.serverImplBuilder.setStatsRecordStartedRpcs(value); } - @Override - protected void setStatsRecordRealTimeMetrics(boolean value) { - super.setStatsRecordRealTimeMetrics(value); + void setStatsRecordRealTimeMetrics(boolean value) { + this.serverImplBuilder.setStatsRecordRealTimeMetrics(value); } /** @@ -562,16 +581,15 @@ public NettyServerBuilder permitKeepAliveWithoutCalls(boolean permit) { return this; } - @Override @CheckReturnValue - protected List buildTransportServers( + List buildTransportServers( List streamTracerFactories) { assertEventLoopsAndChannelType(); ProtocolNegotiator negotiator = protocolNegotiator; if (negotiator == null) { negotiator = sslContext != null - ? ProtocolNegotiators.serverTls(sslContext, this.getExecutorPool()) + ? ProtocolNegotiators.serverTls(sslContext, this.serverImplBuilder.getExecutorPool()) : ProtocolNegotiators.serverPlaintext(); } @@ -580,12 +598,12 @@ protected List buildTransportServers( NettyServer transportServer = new NettyServer( listenAddress, channelFactory, channelOptions, childChannelOptions, bossEventLoopGroupPool, workerEventLoopGroupPool, forceHeapBuffer, negotiator, - streamTracerFactories, getTransportTracerFactory(), maxConcurrentCallsPerConnection, + streamTracerFactories, transportTracerFactory, maxConcurrentCallsPerConnection, autoFlowControl, flowControlWindow, maxMessageSize, maxHeaderListSize, keepAliveTimeInNanos, keepAliveTimeoutInNanos, maxConnectionIdleInNanos, maxConnectionAgeInNanos, maxConnectionAgeGraceInNanos, permitKeepAliveWithoutCalls, permitKeepAliveTimeInNanos, - getChannelz()); + this.serverImplBuilder.getChannelz()); transportServers.add(transportServer); } return Collections.unmodifiableList(transportServers); @@ -605,6 +623,12 @@ void assertEventLoopsAndChannelType() { + "neither should be"); } + NettyServerBuilder setTransportTracerFactory( + TransportTracer.Factory transportTracerFactory) { + this.transportTracerFactory = transportTracerFactory; + return this; + } + @Override public NettyServerBuilder useTransportSecurity(File certChain, File privateKey) { try { diff --git a/okhttp/src/test/java/io/grpc/okhttp/OkHttpTransportTest.java b/okhttp/src/test/java/io/grpc/okhttp/OkHttpTransportTest.java index 8e99b549a31..1f812fe0644 100644 --- a/okhttp/src/test/java/io/grpc/okhttp/OkHttpTransportTest.java +++ b/okhttp/src/test/java/io/grpc/okhttp/OkHttpTransportTest.java @@ -18,12 +18,12 @@ import io.grpc.ServerStreamTracer; import io.grpc.internal.AbstractTransportTest; -import io.grpc.internal.AccessProtectedHack; import io.grpc.internal.ClientTransportFactory; import io.grpc.internal.FakeClock; import io.grpc.internal.GrpcUtil; import io.grpc.internal.InternalServer; import io.grpc.internal.ManagedClientTransport; +import io.grpc.netty.InternalNettyServerBuilder; import io.grpc.netty.NettyServerBuilder; import java.net.InetSocketAddress; import java.util.List; @@ -53,23 +53,21 @@ public void releaseClientFactory() { @Override protected List newServer( List streamTracerFactories) { - return AccessProtectedHack.serverBuilderBuildTransportServer( - NettyServerBuilder - .forPort(0) - .flowControlWindow(AbstractTransportTest.TEST_FLOW_CONTROL_WINDOW), - streamTracerFactories, - fakeClockTransportTracer); + NettyServerBuilder builder = NettyServerBuilder + .forPort(0) + .flowControlWindow(AbstractTransportTest.TEST_FLOW_CONTROL_WINDOW); + InternalNettyServerBuilder.setTransportTracerFactory(builder, fakeClockTransportTracer); + return InternalNettyServerBuilder.buildTransportServers(builder, streamTracerFactories); } @Override protected List newServer( int port, List streamTracerFactories) { - return AccessProtectedHack.serverBuilderBuildTransportServer( - NettyServerBuilder - .forAddress(new InetSocketAddress(port)) - .flowControlWindow(AbstractTransportTest.TEST_FLOW_CONTROL_WINDOW), - streamTracerFactories, - fakeClockTransportTracer); + NettyServerBuilder builder = NettyServerBuilder + .forAddress(new InetSocketAddress(port)) + .flowControlWindow(AbstractTransportTest.TEST_FLOW_CONTROL_WINDOW); + InternalNettyServerBuilder.setTransportTracerFactory(builder, fakeClockTransportTracer); + return InternalNettyServerBuilder.buildTransportServers(builder, streamTracerFactories); } @Override From 3493347581c290b8d8be4627647d60a932572c54 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Wed, 2 Sep 2020 16:49:11 -0400 Subject: [PATCH 19/86] okhttp, testing: remove server builder accessor hacks --- .../io/grpc/internal/AccessProtectedHack.java | 33 ----------------- .../io/grpc/internal/TestingAccessor.java | 35 ------------------- 2 files changed, 68 deletions(-) delete mode 100644 okhttp/src/test/java/io/grpc/internal/AccessProtectedHack.java delete mode 100644 testing/src/main/java/io/grpc/internal/TestingAccessor.java diff --git a/okhttp/src/test/java/io/grpc/internal/AccessProtectedHack.java b/okhttp/src/test/java/io/grpc/internal/AccessProtectedHack.java deleted file mode 100644 index 6b53d821731..00000000000 --- a/okhttp/src/test/java/io/grpc/internal/AccessProtectedHack.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2016 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.internal; - -import io.grpc.ServerStreamTracer; -import java.util.List; - -/** A hack to access protected methods from io.grpc.internal. */ -public final class AccessProtectedHack { - public static List serverBuilderBuildTransportServer( - AbstractServerImplBuilder builder, - List streamTracerFactories, - TransportTracer.Factory transportTracerFactory) { - builder.transportTracerFactory = transportTracerFactory; - return builder.buildTransportServers(streamTracerFactories); - } - - private AccessProtectedHack() {} -} diff --git a/testing/src/main/java/io/grpc/internal/TestingAccessor.java b/testing/src/main/java/io/grpc/internal/TestingAccessor.java deleted file mode 100644 index 92dc114d9c6..00000000000 --- a/testing/src/main/java/io/grpc/internal/TestingAccessor.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2017 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.internal; - -/** - * Test helper that allows accessing package-private stuff. - */ -public final class TestingAccessor { - - /** - * Disable or enable server side census stats features. - */ - public static void setStatsEnabled( - AbstractServerImplBuilder builder, - boolean statsEnabled) { - builder.setStatsEnabled(statsEnabled); - } - - private TestingAccessor() { - } -} From 07012421ad7bd3d46c2f07ec33fa74fd1d105b52 Mon Sep 17 00:00:00 2001 From: Philipp Kern Date: Thu, 3 Sep 2020 23:27:18 +0200 Subject: [PATCH 20/86] core: add accessor for bare method name in MethodDescriptor (#7339) Added getBareMethodName and extractBareMethodName for accessing the unqualified method name. --- .../main/java/io/grpc/MethodDescriptor.java | 27 +++++++++++++++ .../java/io/grpc/MethodDescriptorTest.java | 33 +++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/api/src/main/java/io/grpc/MethodDescriptor.java b/api/src/main/java/io/grpc/MethodDescriptor.java index 08692592e46..c1b8a9ed723 100644 --- a/api/src/main/java/io/grpc/MethodDescriptor.java +++ b/api/src/main/java/io/grpc/MethodDescriptor.java @@ -262,6 +262,17 @@ public String getServiceName() { return serviceName; } + /** + * A convenience method for {@code extractBareMethodName(getFullMethodName())}. + * + * @since 1.32.0 + */ + @Nullable + @ExperimentalApi("https://github.com/grpc/grpc-java/issues/5635") + public String getBareMethodName() { + return extractBareMethodName(fullMethodName); + } + /** * Parse a response payload from the given {@link InputStream}. * @@ -398,6 +409,22 @@ public static String extractFullServiceName(String fullMethodName) { return fullMethodName.substring(0, index); } + /** + * Extract the method name out of a fully qualified method name. May return {@code null} + * if the input is malformed, but you cannot rely on it for the validity of the input. + * + * @since 1.32.0 + */ + @Nullable + @ExperimentalApi("https://github.com/grpc/grpc-java/issues/5635") + public static String extractBareMethodName(String fullMethodName) { + int index = checkNotNull(fullMethodName, "fullMethodName").lastIndexOf('/'); + if (index == -1) { + return null; + } + return fullMethodName.substring(index + 1); + } + /** * Creates a new builder for a {@link MethodDescriptor}. * diff --git a/api/src/test/java/io/grpc/MethodDescriptorTest.java b/api/src/test/java/io/grpc/MethodDescriptorTest.java index 68637648ce4..ec89976a016 100644 --- a/api/src/test/java/io/grpc/MethodDescriptorTest.java +++ b/api/src/test/java/io/grpc/MethodDescriptorTest.java @@ -177,6 +177,39 @@ public void getServiceName_returnsNull() { assertNull(md.getServiceName()); } + @Test + public void getBareMethodName_extractsMethod() { + Marshaller marshaller = TestMethodDescriptors.voidMarshaller(); + MethodDescriptor md = MethodDescriptor.newBuilder(marshaller, marshaller) + .setType(MethodType.UNARY) + .setFullMethodName("foo/bar") + .build(); + + assertEquals("bar", md.getBareMethodName()); + } + + @Test + public void getBareMethodName_returnsNull() { + Marshaller marshaller = TestMethodDescriptors.voidMarshaller(); + MethodDescriptor md = MethodDescriptor.newBuilder(marshaller, marshaller) + .setType(MethodType.UNARY) + .setFullMethodName("foo-bar") + .build(); + + assertNull(md.getBareMethodName()); + } + + @Test + public void getBareMethodName_returnsEmptyStringWithMethodMissing() { + Marshaller marshaller = TestMethodDescriptors.voidMarshaller(); + MethodDescriptor md = MethodDescriptor.newBuilder(marshaller, marshaller) + .setType(MethodType.UNARY) + .setFullMethodName("foo/") + .build(); + + assertTrue(md.getBareMethodName().isEmpty()); + } + @Test public void toBuilderTest() { MethodDescriptor md1 = MethodDescriptor.newBuilder() From 88634ee0d5c1c9db74100183b692dcd4b4633c9a Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Thu, 3 Sep 2020 17:31:30 -0400 Subject: [PATCH 21/86] okhttp: allow disable check authority via internal channel builder --- .../java/io/grpc/okhttp/InternalOkHttpChannelBuilder.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/okhttp/src/main/java/io/grpc/okhttp/InternalOkHttpChannelBuilder.java b/okhttp/src/main/java/io/grpc/okhttp/InternalOkHttpChannelBuilder.java index d328efd7145..a4bd1c03020 100644 --- a/okhttp/src/main/java/io/grpc/okhttp/InternalOkHttpChannelBuilder.java +++ b/okhttp/src/main/java/io/grpc/okhttp/InternalOkHttpChannelBuilder.java @@ -29,5 +29,13 @@ public static void setStatsEnabled(OkHttpChannelBuilder builder, boolean value) builder.setStatsEnabled(value); } + public static void disableCheckAuthority(OkHttpChannelBuilder builder) { + builder.disableCheckAuthority(); + } + + public static void enableCheckAuthority(OkHttpChannelBuilder builder) { + builder.enableCheckAuthority(); + } + private InternalOkHttpChannelBuilder() {} } From 5f9d000a61fe4c304f05004d2fcc6a7f96cf5ab5 Mon Sep 17 00:00:00 2001 From: sanjaypujare Date: Tue, 8 Sep 2020 10:12:55 -0700 Subject: [PATCH 22/86] xds: implement SslContextProviderSupplier to prevent creds leakage (#7364) --- .../java/io/grpc/xds/CdsLoadBalancer.java | 47 +++---- .../main/java/io/grpc/xds/XdsAttributes.java | 12 +- .../internal/sds/SdsProtocolNegotiators.java | 30 ++--- .../xds/internal/sds/SslContextProvider.java | 11 +- .../sds/SslContextProviderSupplier.java | 85 ++++++++++++ .../java/io/grpc/xds/CdsLoadBalancerTest.java | 73 +++++++++-- .../io/grpc/xds/XdsSdsClientServerTest.java | 6 +- .../sds/SdsProtocolNegotiatorsTest.java | 12 +- .../sds/SslContextProviderSupplierTest.java | 122 ++++++++++++++++++ 9 files changed, 328 insertions(+), 70 deletions(-) create mode 100644 xds/src/main/java/io/grpc/xds/internal/sds/SslContextProviderSupplier.java create mode 100644 xds/src/test/java/io/grpc/xds/internal/sds/SslContextProviderSupplierTest.java diff --git a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java index d0b9248ac42..a97846fa54c 100644 --- a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java @@ -40,7 +40,7 @@ import io.grpc.xds.XdsClient.ClusterWatcher; import io.grpc.xds.XdsLogger.XdsLogLevel; import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; -import io.grpc.xds.internal.sds.SslContextProvider; +import io.grpc.xds.internal.sds.SslContextProviderSupplier; import io.grpc.xds.internal.sds.TlsContextManager; import io.grpc.xds.internal.sds.TlsContextManagerImpl; import java.util.ArrayList; @@ -220,32 +220,32 @@ public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { private static final class EdsLoadBalancingHelper extends ForwardingLoadBalancerHelper { private final Helper delegate; - private final AtomicReference sslContextProvider; + private final AtomicReference sslContextProviderSupplier; EdsLoadBalancingHelper(Helper helper, - AtomicReference sslContextProvider) { + AtomicReference sslContextProviderSupplier) { this.delegate = helper; - this.sslContextProvider = sslContextProvider; + this.sslContextProviderSupplier = sslContextProviderSupplier; } @Override public Subchannel createSubchannel(CreateSubchannelArgs createSubchannelArgs) { - if (sslContextProvider.get() != null) { + if (sslContextProviderSupplier.get() != null) { createSubchannelArgs = createSubchannelArgs .toBuilder() .setAddresses( - addUpstreamTlsContext(createSubchannelArgs.getAddresses(), - sslContextProvider.get().getUpstreamTlsContext())) + addSslContextProviderSupplier(createSubchannelArgs.getAddresses(), + sslContextProviderSupplier.get())) .build(); } return delegate.createSubchannel(createSubchannelArgs); } - private static List addUpstreamTlsContext( + private static List addSslContextProviderSupplier( List addresses, - UpstreamTlsContext upstreamTlsContext) { - if (upstreamTlsContext == null || addresses == null) { + SslContextProviderSupplier supplier) { + if (supplier == null || addresses == null) { return addresses; } ArrayList copyList = new ArrayList<>(addresses.size()); @@ -254,7 +254,7 @@ private static List addUpstreamTlsContext( new EquivalentAddressGroup(eag.getAddresses(), eag.getAttributes() .toBuilder() - .set(XdsAttributes.ATTR_UPSTREAM_TLS_CONTEXT, upstreamTlsContext) + .set(XdsAttributes.ATTR_SSL_CONTEXT_PROVIDER_SUPPLIER, supplier) .build() ); copyList.add(eagCopy); @@ -278,7 +278,7 @@ private final class ClusterWatcherImpl implements ClusterWatcher { ClusterWatcherImpl(Helper helper, ResolvedAddresses resolvedAddresses) { this.helper = new EdsLoadBalancingHelper(helper, - new AtomicReference()); + new AtomicReference()); this.resolvedAddresses = resolvedAddresses; } @@ -305,7 +305,7 @@ public void onClusterChanged(ClusterUpdate newUpdate) { /* lrsServerName = */ newUpdate.getLrsServerName(), new PolicySelection(lbProvider, ImmutableMap.of(), lbConfig)); if (isXdsSecurityEnabled()) { - updateSslContextProvider(newUpdate.getUpstreamTlsContext()); + updateSslContextProviderSupplier(newUpdate.getUpstreamTlsContext()); } if (edsBalancer == null) { edsBalancer = lbRegistry.getProvider(EDS_POLICY_NAME).newLoadBalancer(helper); @@ -315,23 +315,24 @@ public void onClusterChanged(ClusterUpdate newUpdate) { } /** For new UpstreamTlsContext value, release old SslContextProvider. */ - private void updateSslContextProvider(UpstreamTlsContext newUpstreamTlsContext) { - SslContextProvider oldSslContextProvider = - helper.sslContextProvider.get(); - if (oldSslContextProvider != null) { - UpstreamTlsContext oldUpstreamTlsContext = oldSslContextProvider.getUpstreamTlsContext(); + private void updateSslContextProviderSupplier(UpstreamTlsContext newUpstreamTlsContext) { + SslContextProviderSupplier oldSslContextProviderSupplier = + helper.sslContextProviderSupplier.get(); + if (oldSslContextProviderSupplier != null) { + UpstreamTlsContext oldUpstreamTlsContext = + oldSslContextProviderSupplier.getUpstreamTlsContext(); if (oldUpstreamTlsContext.equals(newUpstreamTlsContext)) { return; } - tlsContextManager.releaseClientSslContextProvider(oldSslContextProvider); + oldSslContextProviderSupplier.close(); } if (newUpstreamTlsContext != null) { - SslContextProvider newSslContextProvider = - tlsContextManager.findOrCreateClientSslContextProvider(newUpstreamTlsContext); - helper.sslContextProvider.set(newSslContextProvider); + SslContextProviderSupplier newSslContextProviderSupplier = + new SslContextProviderSupplier(newUpstreamTlsContext, tlsContextManager); + helper.sslContextProviderSupplier.set(newSslContextProviderSupplier); } else { - helper.sslContextProvider.set(null); + helper.sslContextProviderSupplier.set(null); } } diff --git a/xds/src/main/java/io/grpc/xds/XdsAttributes.java b/xds/src/main/java/io/grpc/xds/XdsAttributes.java index 57796658636..495f103d281 100644 --- a/xds/src/main/java/io/grpc/xds/XdsAttributes.java +++ b/xds/src/main/java/io/grpc/xds/XdsAttributes.java @@ -21,20 +21,20 @@ import io.grpc.Internal; import io.grpc.NameResolver; import io.grpc.internal.ObjectPool; -import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import io.grpc.xds.LoadStatsManager.LoadStatsStore; +import io.grpc.xds.internal.sds.SslContextProviderSupplier; /** * Special attributes that are only useful to gRPC in the XDS context. */ @Internal public final class XdsAttributes { - /** - * Attribute key for UpstreamTlsContext (used by client) for subchannel. - */ + + /** Attribute key for SslContextProviderSupplier (used from client) for a subchannel. */ @Grpc.TransportAttr - public static final Attributes.Key ATTR_UPSTREAM_TLS_CONTEXT = - Attributes.Key.create("io.grpc.xds.XdsAttributes.upstreamTlsContext"); + public static final Attributes.Key + ATTR_SSL_CONTEXT_PROVIDER_SUPPLIER = + Attributes.Key.create("io.grpc.xds.internal.sds.SslContextProviderSupplier"); @NameResolver.ResolutionResultAttr static final Attributes.Key> XDS_CLIENT_POOL = diff --git a/xds/src/main/java/io/grpc/xds/internal/sds/SdsProtocolNegotiators.java b/xds/src/main/java/io/grpc/xds/internal/sds/SdsProtocolNegotiators.java index d157c796833..d28b5c82f60 100644 --- a/xds/src/main/java/io/grpc/xds/internal/sds/SdsProtocolNegotiators.java +++ b/xds/src/main/java/io/grpc/xds/internal/sds/SdsProtocolNegotiators.java @@ -29,7 +29,6 @@ import io.grpc.netty.NettyChannelBuilder; import io.grpc.netty.ProtocolNegotiationEvent; import io.grpc.xds.EnvoyServerProtoData.DownstreamTlsContext; -import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import io.grpc.xds.XdsAttributes; import io.grpc.xds.XdsClientWrapperForServerSds; import io.netty.channel.ChannelHandler; @@ -116,17 +115,13 @@ public AsciiString scheme() { @Override public ChannelHandler newHandler(GrpcHttp2ConnectionHandler grpcHandler) { - // check if UpstreamTlsContext was passed via attributes - UpstreamTlsContext localUpstreamTlsContext = - grpcHandler.getEagAttributes().get(XdsAttributes.ATTR_UPSTREAM_TLS_CONTEXT); - if (isTlsContextEmpty(localUpstreamTlsContext)) { + // check if SslContextProviderSupplier was passed via attributes + SslContextProviderSupplier localSslContextProviderSupplier = + grpcHandler.getEagAttributes().get(XdsAttributes.ATTR_SSL_CONTEXT_PROVIDER_SUPPLIER); + if (localSslContextProviderSupplier == null) { return InternalProtocolNegotiators.plaintext().newHandler(grpcHandler); } - return new ClientSdsHandler(grpcHandler, localUpstreamTlsContext); - } - - private static boolean isTlsContextEmpty(UpstreamTlsContext upstreamTlsContext) { - return upstreamTlsContext == null || upstreamTlsContext.getCommonTlsContext() == null; + return new ClientSdsHandler(grpcHandler, localSslContextProviderSupplier); } @Override @@ -168,10 +163,11 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { static final class ClientSdsHandler extends InternalProtocolNegotiators.ProtocolNegotiationHandler { private final GrpcHttp2ConnectionHandler grpcHandler; - private final UpstreamTlsContext upstreamTlsContext; + private final SslContextProviderSupplier sslContextProviderSupplier; ClientSdsHandler( - GrpcHttp2ConnectionHandler grpcHandler, UpstreamTlsContext upstreamTlsContext) { + GrpcHttp2ConnectionHandler grpcHandler, + SslContextProviderSupplier sslContextProviderSupplier) { super( // superclass (InternalProtocolNegotiators.ProtocolNegotiationHandler) expects 'next' // handler but we don't have a next handler _yet_. So we "disable" superclass's behavior @@ -184,7 +180,7 @@ public void handlerAdded(ChannelHandlerContext ctx) throws Exception { }); checkNotNull(grpcHandler, "grpcHandler"); this.grpcHandler = grpcHandler; - this.upstreamTlsContext = upstreamTlsContext; + this.sslContextProviderSupplier = sslContextProviderSupplier; } @Override @@ -192,11 +188,7 @@ protected void handlerAdded0(final ChannelHandlerContext ctx) { final BufferReadsHandler bufferReads = new BufferReadsHandler(); ctx.pipeline().addBefore(ctx.name(), null, bufferReads); - final SslContextProvider sslContextProvider = - TlsContextManagerImpl.getInstance() - .findOrCreateClientSslContextProvider(upstreamTlsContext); - - sslContextProvider.addCallback( + sslContextProviderSupplier.updateSslContext( new SslContextProvider.Callback(ctx.executor()) { @Override @@ -212,8 +204,6 @@ public void updateSecret(SslContext sslContext) { ctx.pipeline().addAfter(ctx.name(), null, handler); fireProtocolNegotiationEvent(ctx); ctx.pipeline().remove(bufferReads); - TlsContextManagerImpl.getInstance() - .releaseClientSslContextProvider(sslContextProvider); } @Override diff --git a/xds/src/main/java/io/grpc/xds/internal/sds/SslContextProvider.java b/xds/src/main/java/io/grpc/xds/internal/sds/SslContextProvider.java index 84060f45abf..42608a72348 100644 --- a/xds/src/main/java/io/grpc/xds/internal/sds/SslContextProvider.java +++ b/xds/src/main/java/io/grpc/xds/internal/sds/SslContextProvider.java @@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; +import com.google.common.annotations.VisibleForTesting; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext; import io.grpc.xds.EnvoyServerProtoData.BaseTlsContext; import io.grpc.xds.EnvoyServerProtoData.DownstreamTlsContext; @@ -46,18 +47,22 @@ public abstract class SslContextProvider implements Closeable { protected final BaseTlsContext tlsContext; - abstract static class Callback { + @VisibleForTesting public abstract static class Callback { private final Executor executor; protected Callback(Executor executor) { this.executor = executor; } + @VisibleForTesting public Executor getExecutor() { + return executor; + } + /** Informs callee of new/updated SslContext. */ - abstract void updateSecret(SslContext sslContext); + @VisibleForTesting public abstract void updateSecret(SslContext sslContext); /** Informs callee of an exception that was generated. */ - abstract void onException(Throwable throwable); + @VisibleForTesting protected abstract void onException(Throwable throwable); } protected SslContextProvider(BaseTlsContext tlsContext) { diff --git a/xds/src/main/java/io/grpc/xds/internal/sds/SslContextProviderSupplier.java b/xds/src/main/java/io/grpc/xds/internal/sds/SslContextProviderSupplier.java new file mode 100644 index 00000000000..715b81f35a9 --- /dev/null +++ b/xds/src/main/java/io/grpc/xds/internal/sds/SslContextProviderSupplier.java @@ -0,0 +1,85 @@ +/* + * Copyright 2020 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.xds.internal.sds; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; + +import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; +import io.netty.handler.ssl.SslContext; + +/** + * Enables the CDS policy to initialize this object with the received {@link UpstreamTlsContext} & + * communicate it to the consumer i.e. {@link SdsProtocolNegotiators.ClientSdsProtocolNegotiator} + * to lazily evaluate the {@link SslContextProvider}. The supplier prevents credentials leakage in + * cases where the user is not using xDS credentials but the CDS policy contains a non-default + * {@link UpstreamTlsContext}. + */ +public final class SslContextProviderSupplier implements Closeable { + + private final UpstreamTlsContext upstreamTlsContext; + private final TlsContextManager tlsContextManager; + private SslContextProvider sslContextProvider; + private boolean shutdown; + + public SslContextProviderSupplier( + UpstreamTlsContext upstreamTlsContext, TlsContextManager tlsContextManager) { + this.upstreamTlsContext = upstreamTlsContext; + this.tlsContextManager = tlsContextManager; + } + + public UpstreamTlsContext getUpstreamTlsContext() { + return upstreamTlsContext; + } + + /** Updates SslContext via the passed callback. */ + public synchronized void updateSslContext(final SslContextProvider.Callback callback) { + checkNotNull(callback, "callback"); + checkState(!shutdown, "Supplier is shutdown!"); + if (sslContextProvider == null) { + sslContextProvider = + tlsContextManager.findOrCreateClientSslContextProvider(upstreamTlsContext); + } + // we want to increment the ref-count so call findOrCreate again... + final SslContextProvider toRelease = + tlsContextManager.findOrCreateClientSslContextProvider(upstreamTlsContext); + sslContextProvider.addCallback( + new SslContextProvider.Callback(callback.getExecutor()) { + + @Override + public void updateSecret(SslContext sslContext) { + callback.updateSecret(sslContext); + tlsContextManager.releaseClientSslContextProvider(toRelease); + } + + @Override + public void onException(Throwable throwable) { + callback.onException(throwable); + tlsContextManager.releaseClientSslContextProvider(toRelease); + } + }); + } + + /** Called by {@link io.grpc.xds.CdsLoadBalancer} when upstreamTlsContext changes. */ + @Override + public synchronized void close() { + if (sslContextProvider != null) { + tlsContextManager.releaseClientSslContextProvider(sslContextProvider); + } + shutdown = true; + } +} diff --git a/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java index 9a9bb69241f..77a104262d4 100644 --- a/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java @@ -37,6 +37,7 @@ import static org.mockito.Mockito.verify; import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.MoreExecutors; import io.grpc.Attributes; import io.grpc.ConnectivityState; import io.grpc.EquivalentAddressGroup; @@ -64,7 +65,9 @@ import io.grpc.xds.XdsClient.XdsClientFactory; import io.grpc.xds.internal.sds.CommonTlsContextTestsUtil; import io.grpc.xds.internal.sds.SslContextProvider; +import io.grpc.xds.internal.sds.SslContextProviderSupplier; import io.grpc.xds.internal.sds.TlsContextManager; +import io.netty.handler.ssl.SslContext; import java.net.InetSocketAddress; import java.util.ArrayDeque; import java.util.ArrayList; @@ -391,8 +394,8 @@ public void handleCdsConfigUpdate_withUpstreamTlsContext() { verify(helper, never()) .createSubchannel(any(LoadBalancer.CreateSubchannelArgs.class)); edsLbHelper1.createSubchannel(createSubchannelArgs); - verifyUpstreamTlsContextAttribute(upstreamTlsContext, - createSubchannelArgsCaptor1); + SslContextProviderSupplier sslContextProviderSupplier = + verifySslContextProviderSupplierAttribute(upstreamTlsContext, createSubchannelArgsCaptor1); // update with same upstreamTlsContext reset(mockTlsContextManager); @@ -409,6 +412,29 @@ public void handleCdsConfigUpdate_withUpstreamTlsContext() { verify(mockTlsContextManager, never()).findOrCreateClientSslContextProvider( any(UpstreamTlsContext.class)); + // verify SslContextProviderSupplier.updateSslContext + doReturn(mockSslContextProvider) + .when(mockTlsContextManager) + .findOrCreateClientSslContextProvider(same(upstreamTlsContext)); + sslContextProviderSupplier.updateSslContext( + new SslContextProvider.Callback(MoreExecutors.directExecutor()) { + @Override + public void updateSecret(SslContext sslContext) {} + + @Override + protected void onException(Throwable throwable) {} + }); + verify(mockTlsContextManager, times(2)) + .findOrCreateClientSslContextProvider(eq(upstreamTlsContext)); + verify(mockTlsContextManager, never()) + .releaseClientSslContextProvider(eq(mockSslContextProvider)); + ArgumentCaptor callbackCaptor = ArgumentCaptor.forClass(null); + verify(mockSslContextProvider).addCallback(callbackCaptor.capture()); + SslContextProvider.Callback capturedCallback = callbackCaptor.getValue(); + assertThat(capturedCallback).isNotNull(); + capturedCallback.updateSecret(null); + verify(mockTlsContextManager).releaseClientSslContextProvider(eq(mockSslContextProvider)); + // update with different upstreamTlsContext reset(mockTlsContextManager); reset(helper); @@ -428,12 +454,22 @@ public void handleCdsConfigUpdate_withUpstreamTlsContext() { .build()); verify(mockTlsContextManager).releaseClientSslContextProvider(same(mockSslContextProvider)); - verify(mockTlsContextManager).findOrCreateClientSslContextProvider(same(upstreamTlsContext1)); + verify(mockTlsContextManager, never()) + .findOrCreateClientSslContextProvider(same(upstreamTlsContext1)); ArgumentCaptor createSubchannelArgsCaptor2 = ArgumentCaptor.forClass(null); edsLbHelper1.createSubchannel(createSubchannelArgs); - verifyUpstreamTlsContextAttribute(upstreamTlsContext1, - createSubchannelArgsCaptor2); + SslContextProviderSupplier sslContextProviderSupplier1 = + verifySslContextProviderSupplierAttribute(upstreamTlsContext1, createSubchannelArgsCaptor2); + // initialize sslContextProviderSupplier1 with sslContextProvider + sslContextProviderSupplier1.updateSslContext( + new SslContextProvider.Callback(MoreExecutors.directExecutor()) { + @Override + public void updateSecret(SslContext sslContext) {} + + @Override + protected void onException(Throwable throwable) {} + }); // update with null reset(mockTlsContextManager); @@ -451,7 +487,7 @@ public void handleCdsConfigUpdate_withUpstreamTlsContext() { ArgumentCaptor createSubchannelArgsCaptor3 = ArgumentCaptor.forClass(null); edsLbHelper1.createSubchannel(createSubchannelArgs); - verifyUpstreamTlsContextAttribute(null, + verifySslContextProviderSupplierAttribute(null, createSubchannelArgsCaptor3); LoadBalancer edsLoadBalancer1 = edsLoadBalancers.poll(); @@ -462,23 +498,32 @@ public void handleCdsConfigUpdate_withUpstreamTlsContext() { assertThat(xdsClientPool.xdsClient).isNull(); } - private void verifyUpstreamTlsContextAttribute( + private SslContextProviderSupplier verifySslContextProviderSupplierAttribute( UpstreamTlsContext upstreamTlsContext, ArgumentCaptor createSubchannelArgsCaptor1) { - verify(helper, times(1)).createSubchannel(createSubchannelArgsCaptor1.capture()); + verify(helper).createSubchannel(createSubchannelArgsCaptor1.capture()); CreateSubchannelArgs capturedValue = createSubchannelArgsCaptor1.getValue(); List capturedEagList = capturedValue.getAddresses(); assertThat(capturedEagList.size()).isEqualTo(2); EquivalentAddressGroup capturedEag = capturedEagList.get(0); - UpstreamTlsContext capturedUpstreamTlsContext = - capturedEag.getAttributes().get(XdsAttributes.ATTR_UPSTREAM_TLS_CONTEXT); - assertThat(capturedUpstreamTlsContext).isSameInstanceAs(upstreamTlsContext); + + SslContextProviderSupplier capturedSslContextProviderSupplier = + capturedEag.getAttributes().get(XdsAttributes.ATTR_SSL_CONTEXT_PROVIDER_SUPPLIER); + if (upstreamTlsContext == null) { + assertThat(capturedSslContextProviderSupplier).isNull(); + } else { + assertThat(capturedSslContextProviderSupplier).isNotNull(); + assertThat(capturedSslContextProviderSupplier.getUpstreamTlsContext()) + .isSameInstanceAs(upstreamTlsContext); + } capturedEag = capturedEagList.get(1); - capturedUpstreamTlsContext = - capturedEag.getAttributes().get(XdsAttributes.ATTR_UPSTREAM_TLS_CONTEXT); - assertThat(capturedUpstreamTlsContext).isSameInstanceAs(upstreamTlsContext); + SslContextProviderSupplier capturedSslContextProviderSupplier1 = + capturedEag.getAttributes().get(XdsAttributes.ATTR_SSL_CONTEXT_PROVIDER_SUPPLIER); + assertThat(capturedSslContextProviderSupplier1) + .isSameInstanceAs(capturedSslContextProviderSupplier); assertThat(capturedEag.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL)) .isSameInstanceAs(xdsClientPool); + return capturedSslContextProviderSupplier; } @Test diff --git a/xds/src/test/java/io/grpc/xds/XdsSdsClientServerTest.java b/xds/src/test/java/io/grpc/xds/XdsSdsClientServerTest.java index 346d19398c3..9a94ebddb64 100644 --- a/xds/src/test/java/io/grpc/xds/XdsSdsClientServerTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsSdsClientServerTest.java @@ -48,6 +48,8 @@ import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import io.grpc.xds.internal.sds.CommonTlsContextTestsUtil; import io.grpc.xds.internal.sds.SdsProtocolNegotiators; +import io.grpc.xds.internal.sds.SslContextProviderSupplier; +import io.grpc.xds.internal.sds.TlsContextManagerImpl; import io.grpc.xds.internal.sds.XdsChannelBuilder; import io.grpc.xds.internal.sds.XdsServerBuilder; import io.netty.handler.ssl.NotSslRecordException; @@ -358,7 +360,9 @@ private SimpleServiceGrpc.SimpleServiceBlockingStub getBlockingStub( Attributes attrs = (upstreamTlsContext != null) ? Attributes.newBuilder() - .set(XdsAttributes.ATTR_UPSTREAM_TLS_CONTEXT, upstreamTlsContext) + .set(XdsAttributes.ATTR_SSL_CONTEXT_PROVIDER_SUPPLIER, + new SslContextProviderSupplier( + upstreamTlsContext, TlsContextManagerImpl.getInstance())) .build() : Attributes.EMPTY; fakeNameResolverFactory.setServers( diff --git a/xds/src/test/java/io/grpc/xds/internal/sds/SdsProtocolNegotiatorsTest.java b/xds/src/test/java/io/grpc/xds/internal/sds/SdsProtocolNegotiatorsTest.java index 688f4a4520d..db82bcdb6b9 100644 --- a/xds/src/test/java/io/grpc/xds/internal/sds/SdsProtocolNegotiatorsTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/sds/SdsProtocolNegotiatorsTest.java @@ -163,10 +163,12 @@ public void clientSdsProtocolNegotiatorNewHandler_withTlsContextAttribute() { getCommonTlsContext(/* tlsCertificate= */ null, /* certContext= */ null)); ClientSdsProtocolNegotiator pn = new ClientSdsProtocolNegotiator(); GrpcHttp2ConnectionHandler mockHandler = mock(GrpcHttp2ConnectionHandler.class); + TlsContextManager mockTlsContextManager = mock(TlsContextManager.class); when(mockHandler.getEagAttributes()) .thenReturn( Attributes.newBuilder() - .set(XdsAttributes.ATTR_UPSTREAM_TLS_CONTEXT, upstreamTlsContext) + .set(XdsAttributes.ATTR_SSL_CONTEXT_PROVIDER_SUPPLIER, + new SslContextProviderSupplier(upstreamTlsContext, mockTlsContextManager)) .build()); ChannelHandler newHandler = pn.newHandler(mockHandler); assertThat(newHandler).isNotNull(); @@ -178,8 +180,10 @@ public void clientSdsHandler_addLast() throws IOException { UpstreamTlsContext upstreamTlsContext = buildUpstreamTlsContextFromFilenames(CLIENT_KEY_FILE, CLIENT_PEM_FILE, CA_PEM_FILE); + SslContextProviderSupplier sslContextProviderSupplier = + new SslContextProviderSupplier(upstreamTlsContext, TlsContextManagerImpl.getInstance()); SdsProtocolNegotiators.ClientSdsHandler clientSdsHandler = - new SdsProtocolNegotiators.ClientSdsHandler(grpcHandler, upstreamTlsContext); + new SdsProtocolNegotiators.ClientSdsHandler(grpcHandler, sslContextProviderSupplier); pipeline.addLast(clientSdsHandler); channelHandlerCtx = pipeline.context(clientSdsHandler); assertNotNull(channelHandlerCtx); // clientSdsHandler ctx is non-null since we just added it @@ -334,8 +338,10 @@ public void clientSdsProtocolNegotiatorNewHandler_fireProtocolNegotiationEvent() UpstreamTlsContext upstreamTlsContext = buildUpstreamTlsContextFromFilenames(CLIENT_KEY_FILE, CLIENT_PEM_FILE, CA_PEM_FILE); + SslContextProviderSupplier sslContextProviderSupplier = + new SslContextProviderSupplier(upstreamTlsContext, TlsContextManagerImpl.getInstance()); SdsProtocolNegotiators.ClientSdsHandler clientSdsHandler = - new SdsProtocolNegotiators.ClientSdsHandler(grpcHandler, upstreamTlsContext); + new SdsProtocolNegotiators.ClientSdsHandler(grpcHandler, sslContextProviderSupplier); pipeline.addLast(clientSdsHandler); channelHandlerCtx = pipeline.context(clientSdsHandler); diff --git a/xds/src/test/java/io/grpc/xds/internal/sds/SslContextProviderSupplierTest.java b/xds/src/test/java/io/grpc/xds/internal/sds/SslContextProviderSupplierTest.java new file mode 100644 index 00000000000..881d2f1efd2 --- /dev/null +++ b/xds/src/test/java/io/grpc/xds/internal/sds/SslContextProviderSupplierTest.java @@ -0,0 +1,122 @@ +/* + * Copyright 2020 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.xds.internal.sds; + +import static com.google.common.truth.Truth.assertThat; +import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.CA_PEM_FILE; +import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.CLIENT_KEY_FILE; +import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.CLIENT_PEM_FILE; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import io.grpc.xds.EnvoyServerProtoData; +import io.netty.handler.ssl.SslContext; +import java.util.concurrent.Executor; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +/** + * Unit tests for {@link SslContextProviderSupplier}. + */ +@RunWith(JUnit4.class) +public class SslContextProviderSupplierTest { + + @Mock private TlsContextManager mockTlsContextManager; + private SslContextProviderSupplier supplier; + private SslContextProvider mockSslContextProvider; + private EnvoyServerProtoData.UpstreamTlsContext upstreamTlsContext; + private SslContextProvider.Callback mockCallback; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + } + + private void prepareSupplier() { + upstreamTlsContext = + CommonTlsContextTestsUtil.buildUpstreamTlsContextFromFilenames( + CLIENT_KEY_FILE, CLIENT_PEM_FILE, CA_PEM_FILE); + mockSslContextProvider = mock(SslContextProvider.class); + doReturn(mockSslContextProvider) + .when(mockTlsContextManager) + .findOrCreateClientSslContextProvider(eq(upstreamTlsContext)); + mockCallback = mock(SslContextProvider.Callback.class); + Executor mockExecutor = mock(Executor.class); + doReturn(mockExecutor).when(mockCallback).getExecutor(); + supplier = new SslContextProviderSupplier(upstreamTlsContext, mockTlsContextManager); + supplier.updateSslContext(mockCallback); + } + + @Test + public void get_updateSecret() { + prepareSupplier(); + verify(mockTlsContextManager, times(2)) + .findOrCreateClientSslContextProvider(eq(upstreamTlsContext)); + verify(mockTlsContextManager, times(0)) + .releaseClientSslContextProvider(any(SslContextProvider.class)); + ArgumentCaptor callbackCaptor = ArgumentCaptor.forClass(null); + verify(mockSslContextProvider, times(1)).addCallback(callbackCaptor.capture()); + SslContextProvider.Callback capturedCallback = callbackCaptor.getValue(); + assertThat(capturedCallback).isNotNull(); + SslContext mockSslContext = mock(SslContext.class); + capturedCallback.updateSecret(mockSslContext); + verify(mockCallback, times(1)).updateSecret(eq(mockSslContext)); + verify(mockTlsContextManager, times(1)) + .releaseClientSslContextProvider(eq(mockSslContextProvider)); + SslContextProvider.Callback mockCallback = mock(SslContextProvider.Callback.class); + supplier.updateSslContext(mockCallback); + verify(mockTlsContextManager, times(3)) + .findOrCreateClientSslContextProvider(eq(upstreamTlsContext)); + } + + @Test + public void get_onException() { + prepareSupplier(); + ArgumentCaptor callbackCaptor = ArgumentCaptor.forClass(null); + verify(mockSslContextProvider, times(1)).addCallback(callbackCaptor.capture()); + SslContextProvider.Callback capturedCallback = callbackCaptor.getValue(); + assertThat(capturedCallback).isNotNull(); + capturedCallback.onException(new Exception("test")); + verify(mockTlsContextManager, times(1)) + .releaseClientSslContextProvider(eq(mockSslContextProvider)); + } + + @Test + public void testClose() { + prepareSupplier(); + supplier.close(); + verify(mockTlsContextManager, times(1)) + .releaseClientSslContextProvider(eq(mockSslContextProvider)); + SslContextProvider.Callback mockCallback = mock(SslContextProvider.Callback.class); + try { + supplier.updateSslContext(mockCallback); + Assert.fail("no exception thrown"); + } catch (IllegalStateException expected) { + assertThat(expected).hasMessageThat().isEqualTo("Supplier is shutdown!"); + } + } +} From e61063744030c76794dcdd1863f245e7a202a2f1 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Tue, 8 Sep 2020 14:33:53 -0400 Subject: [PATCH 23/86] okhttp: Keep ChannelBuilder.checkAuthority() for backward compatibility --- .../java/io/grpc/okhttp/OkHttpChannelBuilder.java | 13 +++++++++++++ .../io/grpc/okhttp/OkHttpChannelBuilderTest.java | 11 +++++++++++ 2 files changed, 24 insertions(+) diff --git a/okhttp/src/main/java/io/grpc/okhttp/OkHttpChannelBuilder.java b/okhttp/src/main/java/io/grpc/okhttp/OkHttpChannelBuilder.java index a7759127e62..2b2363781de 100644 --- a/okhttp/src/main/java/io/grpc/okhttp/OkHttpChannelBuilder.java +++ b/okhttp/src/main/java/io/grpc/okhttp/OkHttpChannelBuilder.java @@ -146,6 +146,7 @@ protected OkHttpChannelBuilder(String host, int port) { this(GrpcUtil.authorityFromHostAndPort(host, port)); } + @SuppressWarnings("deprecation") private OkHttpChannelBuilder(String target) { final class OkHttpChannelTransportFactoryBuilder implements ClientTransportFactoryBuilder { @Override @@ -164,6 +165,14 @@ public int getDefaultPort() { managedChannelImplBuilder = new ManagedChannelImplBuilder(target, new OkHttpChannelTransportFactoryBuilder(), new OkHttpChannelDefaultPortProvider()); + + managedChannelImplBuilder.overrideAuthorityChecker( + new io.grpc.internal.ManagedChannelImplBuilder.OverrideAuthorityChecker() { + @Override + public String checkAuthority(String authority) { + return OkHttpChannelBuilder.this.checkAuthority(authority); + } + }); } @Internal @@ -424,6 +433,10 @@ final ClientTransportFactory buildTransportFactory() { useGetForSafeMethods); } + protected String checkAuthority(String authority) { + return GrpcUtil.checkAuthority(authority); + } + final OkHttpChannelBuilder disableCheckAuthority() { this.managedChannelImplBuilder.disableCheckAuthority(); return this; diff --git a/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java b/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java index 4aa1d1aa53c..5be8e95ff80 100644 --- a/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java +++ b/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java @@ -80,6 +80,17 @@ public void failOverrideInvalidAuthority() { builder.overrideAuthority("[invalidauthority"); } + @Test + public void checkAuthorityOverrideAllowsInvalidAuthority() { + OkHttpChannelBuilder builder = new OkHttpChannelBuilder("good", 1234) { + @Override + protected String checkAuthority(String authority) { + return authority; + } + }; + builder.overrideAuthority("[invalidauthority").usePlaintext().buildTransportFactory(); + } + @Test public void disableCheckAuthorityAllowsInvalidAuthority() { OkHttpChannelBuilder builder = new OkHttpChannelBuilder("good", 1234) From b110cf32a003f2825cc0fff9fd979148514a9210 Mon Sep 17 00:00:00 2001 From: sanjaypujare Date: Tue, 8 Sep 2020 17:23:57 -0700 Subject: [PATCH 24/86] xds: add server start semantics as per the server api design (#7371) --- .../xds/XdsClientWrapperForServerSds.java | 86 ++++++- .../xds/internal/sds/ServerWrapperForXds.java | 63 ++++- .../xds/internal/sds/XdsServerBuilder.java | 17 +- .../xds/XdsClientWrapperForServerSdsTest.java | 45 +--- .../XdsClientWrapperForServerSdsTestMisc.java | 127 +++++++-- .../io/grpc/xds/XdsSdsClientServerTest.java | 43 ++-- .../io/grpc/xds/XdsServerBuilderTest.java | 241 ++++++++++++++++-- .../java/io/grpc/xds/XdsServerTestHelper.java | 119 +++++++++ 8 files changed, 630 insertions(+), 111 deletions(-) create mode 100644 xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java diff --git a/xds/src/main/java/io/grpc/xds/XdsClientWrapperForServerSds.java b/xds/src/main/java/io/grpc/xds/XdsClientWrapperForServerSds.java index 95e7cb9e06f..995b1051219 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientWrapperForServerSds.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientWrapperForServerSds.java @@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkState; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; import io.grpc.Internal; import io.grpc.InternalLogId; import io.grpc.Status; @@ -43,7 +44,9 @@ import java.net.UnknownHostException; import java.util.Collections; import java.util.Comparator; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; @@ -70,11 +73,12 @@ public final class XdsClientWrapperForServerSds { private final int port; private ScheduledExecutorService timeService; private XdsClient.ListenerWatcher listenerWatcher; + @VisibleForTesting final Set serverWatchers = new HashSet<>(); /** * Thrown when no suitable management server was found in the bootstrap file. */ - public static final class ManagementServerNotFoundException extends Exception { + public static final class ManagementServerNotFoundException extends IOException { private static final long serialVersionUID = 1; @@ -125,7 +129,7 @@ public boolean hasXdsClient() { } /** Creates an XdsClient and starts a watch. */ - public void createXdsClientAndStart() { + public void createXdsClientAndStart() throws IOException { checkState(xdsClient == null, "start() called more than once"); Bootstrapper.BootstrapInfo bootstrapInfo; List serverList; @@ -135,10 +139,9 @@ public void createXdsClientAndStart() { if (serverList.isEmpty()) { throw new ManagementServerNotFoundException("No management server provided by bootstrap"); } - } catch (IOException | ManagementServerNotFoundException e) { - logger.log(Level.FINE, "Exception reading bootstrap", e); - logger.log(Level.INFO, "Fallback to plaintext for server at port {0}", port); - return; + } catch (IOException e) { + reportError(Status.fromThrowable(e)); + throw e; } Node node = bootstrapInfo.getNode(); timeService = SharedResourceHolder.get(timeServiceResource); @@ -165,23 +168,22 @@ public void start(XdsClient xdsClient) { new XdsClient.ListenerWatcher() { @Override public void onListenerChanged(XdsClient.ListenerUpdate update) { - logger.log( - Level.INFO, - "Setting myListener from ConfigUpdate listener: {0}", - update.getListener()); curListener = update.getListener(); + reportSuccess(); } @Override public void onResourceDoesNotExist(String resourceName) { - logger.log(Level.INFO, "Resource {0} is unavailable", resourceName); + logger.log(Level.WARNING, "Resource {0} is unavailable", resourceName); curListener = null; + reportError(Status.NOT_FOUND.withDescription(resourceName)); } @Override public void onError(Status error) { - // TODO(sanjaypujare): Implement logic for other cases based on final design. - logger.log(Level.SEVERE, "ListenerWatcher in XdsClientWrapperForServerSds: {0}", error); + logger.log( + Level.WARNING, "ListenerWatcher in XdsClientWrapperForServerSds: {0}", error); + reportError(error); } }; xdsClient.watchListenerData(port, listenerWatcher); @@ -202,6 +204,14 @@ public DownstreamTlsContext getDownstreamTlsContext(Channel channel) { checkState( port == localInetAddr.getPort(), "Channel localAddress port does not match requested listener port"); + return getDownstreamTlsContext(localInetAddr); + } + return null; + } + + private DownstreamTlsContext getDownstreamTlsContext(InetSocketAddress localInetAddr) { + checkNotNull(localInetAddr, "localInetAddr"); + if (curListener != null) { List filterChains = curListener.getFilterChains(); FilterChainComparator comparator = new FilterChainComparator(localInetAddr); FilterChain bestMatch = @@ -213,11 +223,61 @@ public DownstreamTlsContext getDownstreamTlsContext(Channel channel) { return null; } + /** Adds a {@link ServerWatcher} to the list. */ + public void addServerWatcher(ServerWatcher serverWatcher) { + checkNotNull(serverWatcher, "serverWatcher"); + synchronized (serverWatchers) { + serverWatchers.add(serverWatcher); + } + if (curListener != null) { + serverWatcher.onSuccess(getDownstreamTlsContext(new InetSocketAddress(port))); + } + } + + /** Removes a {@link ServerWatcher} from the list. */ + public void removeServerWatcher(ServerWatcher serverWatcher) { + checkNotNull(serverWatcher, "serverWatcher"); + synchronized (serverWatchers) { + serverWatchers.remove(serverWatcher); + } + } + + private Set getServerWatchers() { + synchronized (serverWatchers) { + return ImmutableSet.copyOf(serverWatchers); + } + } + + private void reportError(Status status) { + for (ServerWatcher watcher : getServerWatchers()) { + watcher.onError(status); + } + } + + private void reportSuccess() { + DownstreamTlsContext downstreamTlsContext = + getDownstreamTlsContext(new InetSocketAddress(port)); + for (ServerWatcher watcher : getServerWatchers()) { + watcher.onSuccess(downstreamTlsContext); + } + } + + @VisibleForTesting XdsClient.ListenerWatcher getListenerWatcher() { return listenerWatcher; } + /** Watcher interface for the clients of this class. */ + public interface ServerWatcher { + + /** Called to report errors from the control plane including "not found". */ + void onError(Status error); + + /** Called to report successful receipt of server config. */ + void onSuccess(DownstreamTlsContext downstreamTlsContext); + } + private static final class FilterChainComparator implements Comparator { private final InetSocketAddress localAddress; diff --git a/xds/src/main/java/io/grpc/xds/internal/sds/ServerWrapperForXds.java b/xds/src/main/java/io/grpc/xds/internal/sds/ServerWrapperForXds.java index 09bd8793056..cda94cf210b 100644 --- a/xds/src/main/java/io/grpc/xds/internal/sds/ServerWrapperForXds.java +++ b/xds/src/main/java/io/grpc/xds/internal/sds/ServerWrapperForXds.java @@ -17,15 +17,23 @@ package io.grpc.xds.internal.sds; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; import com.google.common.annotations.VisibleForTesting; +import com.google.common.util.concurrent.SettableFuture; import io.grpc.Server; import io.grpc.ServerServiceDefinition; +import io.grpc.Status; +import io.grpc.xds.EnvoyServerProtoData; import io.grpc.xds.XdsClientWrapperForServerSds; import java.io.IOException; import java.net.SocketAddress; import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.annotation.Nullable; /** * Wraps a {@link Server} delegate and {@link XdsClientWrapperForServerSds} and intercepts {@link @@ -36,22 +44,71 @@ public final class ServerWrapperForXds extends Server { private final Server delegate; private final XdsClientWrapperForServerSds xdsClientWrapperForServerSds; - - ServerWrapperForXds(Server delegate, XdsClientWrapperForServerSds xdsClientWrapperForServerSds) { + @Nullable XdsServerBuilder.ErrorNotifier errorNotifier; + @Nullable XdsClientWrapperForServerSds.ServerWatcher serverWatcher; + private AtomicBoolean started = new AtomicBoolean(); + + ServerWrapperForXds( + Server delegate, + XdsClientWrapperForServerSds xdsClientWrapperForServerSds, + @Nullable XdsServerBuilder.ErrorNotifier errorNotifier) { this.delegate = checkNotNull(delegate, "delegate"); this.xdsClientWrapperForServerSds = checkNotNull(xdsClientWrapperForServerSds, "xdsClientWrapperForServerSds"); + this.errorNotifier = errorNotifier; } @Override public Server start() throws IOException { - delegate.start(); + checkState(started.compareAndSet(false, true), "Already started"); + Future future = addServerWatcher(); if (!xdsClientWrapperForServerSds.hasXdsClient()) { xdsClientWrapperForServerSds.createXdsClientAndStart(); } + try { + future.get(); + } catch (InterruptedException | ExecutionException ex) { + removeServerWatcher(); + if (ex instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new RuntimeException(ex); + } + delegate.start(); return this; } + private Future addServerWatcher() { + final SettableFuture settableFuture = + SettableFuture.create(); + serverWatcher = + new XdsClientWrapperForServerSds.ServerWatcher() { + @Override + public void onError(Status error) { + if (errorNotifier != null) { + errorNotifier.onError(error); + } + } + + @Override + public void onSuccess(EnvoyServerProtoData.DownstreamTlsContext downstreamTlsContext) { + removeServerWatcher(); + settableFuture.set(downstreamTlsContext); + } + }; + xdsClientWrapperForServerSds.addServerWatcher(serverWatcher); + return settableFuture; + } + + private void removeServerWatcher() { + synchronized (xdsClientWrapperForServerSds) { + if (serverWatcher != null) { + xdsClientWrapperForServerSds.removeServerWatcher(serverWatcher); + serverWatcher = null; + } + } + } + @Override public Server shutdown() { xdsClientWrapperForServerSds.shutdown(); diff --git a/xds/src/main/java/io/grpc/xds/internal/sds/XdsServerBuilder.java b/xds/src/main/java/io/grpc/xds/internal/sds/XdsServerBuilder.java index 47315797a05..0f767744f89 100644 --- a/xds/src/main/java/io/grpc/xds/internal/sds/XdsServerBuilder.java +++ b/xds/src/main/java/io/grpc/xds/internal/sds/XdsServerBuilder.java @@ -27,6 +27,7 @@ import io.grpc.ServerServiceDefinition; import io.grpc.ServerStreamTracer; import io.grpc.ServerTransportFilter; +import io.grpc.Status; import io.grpc.netty.InternalProtocolNegotiator.ProtocolNegotiator; import io.grpc.netty.NettyServerBuilder; import io.grpc.xds.internal.sds.SdsProtocolNegotiators.ServerSdsProtocolNegotiator; @@ -45,6 +46,7 @@ public final class XdsServerBuilder extends ServerBuilder { private final NettyServerBuilder delegate; private final int port; private ProtocolNegotiator fallbackProtocolNegotiator; + private ErrorNotifier errorNotifier; private XdsServerBuilder(NettyServerBuilder nettyDelegate, int port) { this.delegate = nettyDelegate; @@ -129,6 +131,12 @@ public XdsServerBuilder fallbackProtocolNegotiator( return this; } + /** Set the {@link ErrorNotifier}. Pass null to unset a previously set value. */ + public XdsServerBuilder errorNotifier(ErrorNotifier errorNotifier) { + this.errorNotifier = errorNotifier; + return this; + } + /** Creates a gRPC server builder for the given port. */ public static XdsServerBuilder forPort(int port) { NettyServerBuilder nettyDelegate = NettyServerBuilder.forAddress(new InetSocketAddress(port)); @@ -150,6 +158,13 @@ public Server build() { public ServerWrapperForXds buildServer(ServerSdsProtocolNegotiator serverProtocolNegotiator) { delegate.protocolNegotiator(serverProtocolNegotiator); return new ServerWrapperForXds( - delegate.build(), serverProtocolNegotiator.getXdsClientWrapperForServerSds()); + delegate.build(), serverProtocolNegotiator.getXdsClientWrapperForServerSds(), + errorNotifier); + } + + /** Watcher to receive error notifications from xDS control plane during {@code start()}. */ + public interface ErrorNotifier { + + void onError(Status error); } } diff --git a/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTest.java b/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTest.java index 7e099de6c34..9237981b9a4 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTest.java @@ -22,7 +22,6 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import com.google.common.base.Strings; import io.grpc.xds.EnvoyServerProtoData.DownstreamTlsContext; import io.grpc.xds.internal.sds.CommonTlsContextTestsUtil; import io.netty.channel.Channel; @@ -31,7 +30,6 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; -import java.util.ArrayList; import java.util.Arrays; import org.junit.After; import org.junit.Before; @@ -193,7 +191,7 @@ public void commonFilterChainMatchTest() InetSocketAddress localAddress = new InetSocketAddress(ipLocalAddress, PORT); when(channel.localAddress()).thenReturn(localAddress); EnvoyServerProtoData.Listener listener = - buildTestListener( + XdsServerTestHelper.buildTestListener( "listener1", "10.1.2.3", destPort1, @@ -212,45 +210,4 @@ public void commonFilterChainMatchTest() assertThat(downstreamTlsContext).isSameInstanceAs(tlsContexts[expectedIndex]); } - static EnvoyServerProtoData.Listener buildTestListener( - String name, - String address, - int destPort1, - int destPort2, - String addressPrefix11, - String addressPrefix12, - String addressPrefix21, - String addressPrefix22, - DownstreamTlsContext tlsContext1, - DownstreamTlsContext tlsContext2) { - EnvoyServerProtoData.FilterChainMatch filterChainMatch1 = - destPort1 > 0 ? buildFilterChainMatch(destPort1, addressPrefix11, addressPrefix12) : null; - EnvoyServerProtoData.FilterChainMatch filterChainMatch2 = - destPort2 > 0 ? buildFilterChainMatch(destPort2, addressPrefix21, addressPrefix22) : null; - EnvoyServerProtoData.FilterChain filterChain1 = - new EnvoyServerProtoData.FilterChain(filterChainMatch1, tlsContext1); - EnvoyServerProtoData.FilterChain filterChain2 = - new EnvoyServerProtoData.FilterChain(filterChainMatch2, tlsContext2); - EnvoyServerProtoData.Listener listener = - new EnvoyServerProtoData.Listener(name, address, Arrays.asList(filterChain1, filterChain2)); - return listener; - } - - static EnvoyServerProtoData.FilterChainMatch buildFilterChainMatch( - int destPort, String... addressPrefix) { - ArrayList prefixRanges = new ArrayList<>(); - for (String address : addressPrefix) { - if (!Strings.isNullOrEmpty(address)) { - prefixRanges.add(new EnvoyServerProtoData.CidrRange(address, 32)); - } - } - return new EnvoyServerProtoData.FilterChainMatch( - destPort, prefixRanges, Arrays.asList()); - } - - static EnvoyServerProtoData.FilterChainMatch buildFilterChainMatch( - int destPort, EnvoyServerProtoData.CidrRange... prefixRanges) { - return new EnvoyServerProtoData.FilterChainMatch( - destPort, Arrays.asList(prefixRanges), Arrays.asList()); - } } diff --git a/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTestMisc.java b/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTestMisc.java index 920f6b23d8e..d0e34b3fe7d 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTestMisc.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTestMisc.java @@ -20,11 +20,17 @@ import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.same; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import io.grpc.Status; import io.grpc.inprocess.InProcessSocketAddress; import io.grpc.xds.EnvoyServerProtoData.DownstreamTlsContext; +import io.grpc.xds.internal.sds.CommonTlsContextTestsUtil; import io.netty.channel.Channel; import java.io.IOException; import java.net.InetAddress; @@ -51,12 +57,12 @@ public class XdsClientWrapperForServerSdsTestMisc { @Mock private Channel channel; private XdsClientWrapperForServerSds xdsClientWrapperForServerSds; + private XdsClient.ListenerWatcher registeredWatcher; @Before public void setUp() throws IOException { MockitoAnnotations.initMocks(this); xdsClientWrapperForServerSds = new XdsClientWrapperForServerSds(PORT); - xdsClientWrapperForServerSds.start(xdsClient); } @After @@ -66,14 +72,18 @@ public void tearDown() { @Test public void verifyListenerWatcherRegistered() { + registeredWatcher = + XdsServerTestHelper.startAndGetWatcher(xdsClientWrapperForServerSds, xdsClient, PORT); verify(xdsClient).watchListenerData(eq(PORT), any(XdsClient.ListenerWatcher.class)); } @Test public void nonInetSocketAddress_expectException() { + registeredWatcher = + XdsServerTestHelper.startAndGetWatcher(xdsClientWrapperForServerSds, xdsClient, PORT); try { DownstreamTlsContext unused = - commonTestPrep(new InProcessSocketAddress("test1")); + sendListenerUpdate(new InProcessSocketAddress("test1"), null, null); fail("exception expected"); } catch (IllegalStateException expected) { assertThat(expected) @@ -84,10 +94,12 @@ public void nonInetSocketAddress_expectException() { @Test public void nonMatchingPort_expectException() throws UnknownHostException { + registeredWatcher = + XdsServerTestHelper.startAndGetWatcher(xdsClientWrapperForServerSds, xdsClient, PORT); try { InetAddress ipLocalAddress = InetAddress.getByName("10.1.2.3"); InetSocketAddress localAddress = new InetSocketAddress(ipLocalAddress, PORT + 1); - DownstreamTlsContext unused = commonTestPrep(localAddress); + DownstreamTlsContext unused = sendListenerUpdate(localAddress, null, null); fail("exception expected"); } catch (IllegalStateException expected) { assertThat(expected) @@ -98,6 +110,8 @@ public void nonMatchingPort_expectException() throws UnknownHostException { @Test public void emptyFilterChain_expectNull() throws UnknownHostException { + registeredWatcher = + XdsServerTestHelper.startAndGetWatcher(xdsClientWrapperForServerSds, xdsClient, PORT); InetAddress ipLocalAddress = InetAddress.getByName("10.1.2.3"); InetSocketAddress localAddress = new InetSocketAddress(ipLocalAddress, PORT); ArgumentCaptor listenerWatcherCaptor = ArgumentCaptor.forClass(null); @@ -105,26 +119,107 @@ public void emptyFilterChain_expectNull() throws UnknownHostException { XdsClient.ListenerWatcher registeredWatcher = listenerWatcherCaptor.getValue(); when(channel.localAddress()).thenReturn(localAddress); EnvoyServerProtoData.Listener listener = - new EnvoyServerProtoData.Listener("listener1", - "10.1.2.3", Collections.emptyList()); + new EnvoyServerProtoData.Listener( + "listener1", "10.1.2.3", Collections.emptyList()); XdsClient.ListenerUpdate listenerUpdate = - XdsClient.ListenerUpdate.newBuilder().setListener(listener).build(); + XdsClient.ListenerUpdate.newBuilder().setListener(listener).build(); registeredWatcher.onListenerChanged(listenerUpdate); DownstreamTlsContext tlsContext = xdsClientWrapperForServerSds.getDownstreamTlsContext(channel); assertThat(tlsContext).isNull(); } - private DownstreamTlsContext commonTestPrep(SocketAddress localAddress) { - ArgumentCaptor listenerWatcherCaptor = ArgumentCaptor.forClass(null); - verify(xdsClient).watchListenerData(eq(PORT), listenerWatcherCaptor.capture()); - XdsClient.ListenerWatcher registeredWatcher = listenerWatcherCaptor.getValue(); + @Test + public void registerServerWatcher() throws UnknownHostException { + registeredWatcher = + XdsServerTestHelper.startAndGetWatcher(xdsClientWrapperForServerSds, xdsClient, PORT); + XdsClientWrapperForServerSds.ServerWatcher mockServerWatcher = + mock(XdsClientWrapperForServerSds.ServerWatcher.class); + xdsClientWrapperForServerSds.addServerWatcher(mockServerWatcher); + InetAddress ipLocalAddress = InetAddress.getByName("10.1.2.3"); + InetSocketAddress localAddress = new InetSocketAddress(ipLocalAddress, PORT); + EnvoyServerProtoData.DownstreamTlsContext tlsContext = + CommonTlsContextTestsUtil.buildTestInternalDownstreamTlsContext("CERT1", "VA1"); + verify(mockServerWatcher, never()) + .onSuccess(any(EnvoyServerProtoData.DownstreamTlsContext.class)); + DownstreamTlsContext returnedTlsContext = sendListenerUpdate(localAddress, tlsContext, null); + assertThat(returnedTlsContext).isSameInstanceAs(tlsContext); + verify(mockServerWatcher).onSuccess(same(tlsContext)); + xdsClientWrapperForServerSds.removeServerWatcher(mockServerWatcher); + } + + @Test + public void registerServerWatcher_afterListenerUpdate() throws UnknownHostException { + registeredWatcher = + XdsServerTestHelper.startAndGetWatcher(xdsClientWrapperForServerSds, xdsClient, PORT); + InetAddress ipLocalAddress = InetAddress.getByName("10.1.2.3"); + InetSocketAddress localAddress = new InetSocketAddress(ipLocalAddress, PORT); + EnvoyServerProtoData.DownstreamTlsContext tlsContext = + CommonTlsContextTestsUtil.buildTestInternalDownstreamTlsContext("CERT1", "VA1"); + DownstreamTlsContext returnedTlsContext = sendListenerUpdate(localAddress, tlsContext, null); + assertThat(returnedTlsContext).isSameInstanceAs(tlsContext); + XdsClientWrapperForServerSds.ServerWatcher mockServerWatcher = + mock(XdsClientWrapperForServerSds.ServerWatcher.class); + xdsClientWrapperForServerSds.addServerWatcher(mockServerWatcher); + verify(mockServerWatcher).onSuccess(same(tlsContext)); + } + + @Test + public void registerServerWatcher_notifyError() throws UnknownHostException { + registeredWatcher = + XdsServerTestHelper.startAndGetWatcher(xdsClientWrapperForServerSds, xdsClient, PORT); + XdsClientWrapperForServerSds.ServerWatcher mockServerWatcher = + mock(XdsClientWrapperForServerSds.ServerWatcher.class); + xdsClientWrapperForServerSds.addServerWatcher(mockServerWatcher); + registeredWatcher.onError(Status.INTERNAL); + verify(mockServerWatcher).onError(eq(Status.INTERNAL)); + reset(mockServerWatcher); + registeredWatcher.onResourceDoesNotExist("not-found Error"); + ArgumentCaptor argCaptor = ArgumentCaptor.forClass(null); + verify(mockServerWatcher).onError(argCaptor.capture()); + Status captured = argCaptor.getValue(); + assertThat(captured.getCode()).isEqualTo(Status.Code.NOT_FOUND); + assertThat(captured.getDescription()).isEqualTo("not-found Error"); + InetAddress ipLocalAddress = InetAddress.getByName("10.1.2.3"); + InetSocketAddress localAddress = new InetSocketAddress(ipLocalAddress, PORT); + EnvoyServerProtoData.DownstreamTlsContext tlsContext = + CommonTlsContextTestsUtil.buildTestInternalDownstreamTlsContext("CERT1", "VA1"); + verify(mockServerWatcher, never()) + .onSuccess(any(EnvoyServerProtoData.DownstreamTlsContext.class)); + DownstreamTlsContext returnedTlsContext = sendListenerUpdate(localAddress, tlsContext, null); + assertThat(returnedTlsContext).isSameInstanceAs(tlsContext); + verify(mockServerWatcher).onSuccess(same(tlsContext)); + } + + @Test + public void startXdsClient_expectException() { + XdsClientWrapperForServerSds.ServerWatcher mockServerWatcher = + mock(XdsClientWrapperForServerSds.ServerWatcher.class); + xdsClientWrapperForServerSds.addServerWatcher(mockServerWatcher); + try { + xdsClientWrapperForServerSds.createXdsClientAndStart(); + fail("exception expected"); + } catch (IOException expected) { + assertThat(expected) + .hasMessageThat() + .contains("Environment variable GRPC_XDS_BOOTSTRAP not defined"); + } + ArgumentCaptor argCaptor = ArgumentCaptor.forClass(null); + verify(mockServerWatcher).onError(argCaptor.capture()); + Status captured = argCaptor.getValue(); + assertThat(captured.getCode()).isEqualTo(Status.Code.UNKNOWN); + assertThat(captured.getCause()).isInstanceOf(IOException.class); + assertThat(captured.getCause()) + .hasMessageThat() + .contains("Environment variable GRPC_XDS_BOOTSTRAP not defined"); + } + + private DownstreamTlsContext sendListenerUpdate( + SocketAddress localAddress, + DownstreamTlsContext tlsContext1, + DownstreamTlsContext tlsContext2) { when(channel.localAddress()).thenReturn(localAddress); - EnvoyServerProtoData.Listener listener = - XdsClientWrapperForServerSdsTest.buildTestListener( - "listener1", "10.1.2.3", PORT, PORT, null, null, null, null, null, null); - XdsClient.ListenerUpdate listenerUpdate = - XdsClient.ListenerUpdate.newBuilder().setListener(listener).build(); - registeredWatcher.onListenerChanged(listenerUpdate); + XdsServerTestHelper.generateListenerUpdate( + registeredWatcher, PORT, tlsContext1, tlsContext2); return xdsClientWrapperForServerSds.getDownstreamTlsContext(channel); } } diff --git a/xds/src/test/java/io/grpc/xds/XdsSdsClientServerTest.java b/xds/src/test/java/io/grpc/xds/XdsSdsClientServerTest.java index 9a94ebddb64..3ccf551756e 100644 --- a/xds/src/test/java/io/grpc/xds/XdsSdsClientServerTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsSdsClientServerTest.java @@ -17,7 +17,7 @@ package io.grpc.xds; import static com.google.common.truth.Truth.assertThat; -import static io.grpc.xds.XdsClientWrapperForServerSdsTest.buildFilterChainMatch; +import static io.grpc.xds.XdsServerTestHelper.buildFilterChainMatch; import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.BAD_CLIENT_KEY_FILE; import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.BAD_CLIENT_PEM_FILE; import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.BAD_SERVER_KEY_FILE; @@ -28,6 +28,7 @@ import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.SERVER_1_KEY_FILE; import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.SERVER_1_PEM_FILE; import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; import com.google.common.collect.ImmutableList; import io.grpc.Attributes; @@ -56,7 +57,6 @@ import java.io.IOException; import java.net.Inet4Address; import java.net.InetSocketAddress; -import java.net.ServerSocket; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; @@ -83,7 +83,7 @@ public class XdsSdsClientServerTest { @Before public void setUp() throws IOException { - port = findFreePort(); + port = XdsServerTestHelper.findFreePort(); } @After @@ -285,7 +285,7 @@ private XdsClient.ListenerWatcher performMtlsTestAndGetListenerWatcher( XdsClientWrapperForServerSdsTest.createXdsClientWrapperForServerSds( port, /* downstreamTlsContext= */ downstreamTlsContext); buildServerWithFallbackProtocolNegotiator(xdsClientWrapperForServerSds, - InternalProtocolNegotiators.serverPlaintext()); + InternalProtocolNegotiators.serverPlaintext(), downstreamTlsContext); XdsClient.ListenerWatcher listenerWatcher = xdsClientWrapperForServerSds.getListenerWatcher(); @@ -304,36 +304,41 @@ private void buildServerWithTlsContext(DownstreamTlsContext downstreamTlsContext private void buildServerWithTlsContext(DownstreamTlsContext downstreamTlsContext, ProtocolNegotiator fallbackProtocolNegotiator) throws IOException { - final XdsClientWrapperForServerSds xdsClientWrapperForServerSds = - XdsClientWrapperForServerSdsTest.createXdsClientWrapperForServerSds( - port, /* downstreamTlsContext= */ downstreamTlsContext); - buildServerWithFallbackProtocolNegotiator(xdsClientWrapperForServerSds, - fallbackProtocolNegotiator); + XdsClient mockXdsClient = mock(XdsClient.class); + XdsClientWrapperForServerSds xdsClientWrapperForServerSds = + new XdsClientWrapperForServerSds(port); + xdsClientWrapperForServerSds.start(mockXdsClient); + buildServerWithFallbackProtocolNegotiator( + xdsClientWrapperForServerSds, fallbackProtocolNegotiator, downstreamTlsContext); } private void buildServerWithFallbackProtocolNegotiator( XdsClientWrapperForServerSds xdsClientWrapperForServerSds, - ProtocolNegotiator fallbackProtocolNegotiator) throws IOException { + ProtocolNegotiator fallbackProtocolNegotiator, + DownstreamTlsContext downstreamTlsContext) + throws IOException { SdsProtocolNegotiators.ServerSdsProtocolNegotiator serverSdsProtocolNegotiator = new SdsProtocolNegotiators.ServerSdsProtocolNegotiator(xdsClientWrapperForServerSds, fallbackProtocolNegotiator); - buildServer(port, serverSdsProtocolNegotiator); + buildServer( + port, serverSdsProtocolNegotiator, xdsClientWrapperForServerSds, downstreamTlsContext); } private void buildServer( - int port, SdsProtocolNegotiators.ServerSdsProtocolNegotiator serverSdsProtocolNegotiator) + int port, + SdsProtocolNegotiators.ServerSdsProtocolNegotiator serverSdsProtocolNegotiator, + XdsClientWrapperForServerSds xdsClientWrapperForServerSds, + DownstreamTlsContext downstreamTlsContext) throws IOException { XdsServerBuilder builder = XdsServerBuilder.forPort(port).addService(new SimpleServiceImpl()); + XdsServerTestHelper.generateListenerUpdate( + xdsClientWrapperForServerSds.getListenerWatcher(), + port, + downstreamTlsContext, + /* tlsContext2= */null); cleanupRule.register(builder.buildServer(serverSdsProtocolNegotiator)).start(); } - private static int findFreePort() throws IOException { - try (ServerSocket socket = new ServerSocket(0)) { - socket.setReuseAddress(true); - return socket.getLocalPort(); - } - } - static EnvoyServerProtoData.Listener buildListener( String name, String address, int port, DownstreamTlsContext tlsContext) { EnvoyServerProtoData.FilterChainMatch filterChainMatch = buildFilterChainMatch(port, address); diff --git a/xds/src/test/java/io/grpc/xds/XdsServerBuilderTest.java b/xds/src/test/java/io/grpc/xds/XdsServerBuilderTest.java index 8c67d52c194..bab78b22647 100644 --- a/xds/src/test/java/io/grpc/xds/XdsServerBuilderTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsServerBuilderTest.java @@ -16,20 +16,39 @@ package io.grpc.xds; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import com.google.common.util.concurrent.SettableFuture; +import io.grpc.Status; import io.grpc.netty.InternalProtocolNegotiators; +import io.grpc.testing.GrpcCleanupRule; +import io.grpc.xds.internal.sds.CommonTlsContextTestsUtil; import io.grpc.xds.internal.sds.SdsProtocolNegotiators.ServerSdsProtocolNegotiator; import io.grpc.xds.internal.sds.ServerWrapperForXds; import io.grpc.xds.internal.sds.XdsServerBuilder; import java.io.IOException; +import java.net.InetSocketAddress; import java.net.ServerSocket; +import java.net.SocketAddress; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; +import org.mockito.ArgumentCaptor; /** * Unit tests for {@link XdsServerBuilder}. @@ -37,28 +56,220 @@ @RunWith(JUnit4.class) public class XdsServerBuilderTest { - @Test - public void xdsServer_callsShutdown() throws IOException, InterruptedException { - int port = findFreePort(); + @Rule public final GrpcCleanupRule cleanupRule = new GrpcCleanupRule(); + private XdsClient mockXdsClient; + private ServerWrapperForXds xdsServer; + private XdsClient.ListenerWatcher listenerWatcher; + private int port; + private XdsClientWrapperForServerSds xdsClientWrapperForServerSds; + + private void buildServer( + XdsServerBuilder.ErrorNotifier errorNotifier, boolean injectMockXdsClient) + throws IOException { + port = XdsServerTestHelper.findFreePort(); XdsServerBuilder builder = XdsServerBuilder.forPort(port); - XdsClient mockXdsClient = mock(XdsClient.class); - XdsClientWrapperForServerSds xdsClientWrapperForServerSds = - new XdsClientWrapperForServerSds(port); - xdsClientWrapperForServerSds.start(mockXdsClient); + if (errorNotifier != null) { + builder = builder.errorNotifier(errorNotifier); + } + xdsClientWrapperForServerSds = new XdsClientWrapperForServerSds(port); + if (injectMockXdsClient) { + mockXdsClient = mock(XdsClient.class); + listenerWatcher = + XdsServerTestHelper.startAndGetWatcher(xdsClientWrapperForServerSds, mockXdsClient, port); + } ServerSdsProtocolNegotiator serverSdsProtocolNegotiator = - new ServerSdsProtocolNegotiator(xdsClientWrapperForServerSds, - InternalProtocolNegotiators.serverPlaintext()); - ServerWrapperForXds xdsServer = builder.buildServer(serverSdsProtocolNegotiator); - xdsServer.start(); + new ServerSdsProtocolNegotiator( + xdsClientWrapperForServerSds, InternalProtocolNegotiators.serverPlaintext()); + xdsServer = cleanupRule.register(builder.buildServer(serverSdsProtocolNegotiator)); + } + + private void verifyServer( + Future future, XdsServerBuilder.ErrorNotifier mockErrorNotifier) + throws InterruptedException, ExecutionException, TimeoutException { + if (future != null) { + Throwable exception = future.get(5, TimeUnit.SECONDS); + assertThat(exception).isNull(); + } + List list = xdsServer.getListenSockets(); + assertThat(list).hasSize(1); + InetSocketAddress socketAddress = (InetSocketAddress) list.get(0); + assertThat(socketAddress.getAddress().isAnyLocalAddress()).isTrue(); + assertThat(socketAddress.getPort()).isEqualTo(port); + if (mockErrorNotifier != null) { + verify(mockErrorNotifier, never()).onError(any(Status.class)); + } + assertThat(xdsClientWrapperForServerSds.serverWatchers).isEmpty(); + } + + private void verifyShutdown() throws InterruptedException { xdsServer.shutdown(); xdsServer.awaitTermination(500L, TimeUnit.MILLISECONDS); verify(mockXdsClient, times(1)).shutdown(); } - private static int findFreePort() throws IOException { - try (ServerSocket socket = new ServerSocket(0)) { - socket.setReuseAddress(true); - return socket.getLocalPort(); + private Future startServerAsync() throws InterruptedException { + final SettableFuture settableFuture = SettableFuture.create(); + Executors.newSingleThreadExecutor().execute(new Runnable() { + @Override + public void run() { + try { + xdsServer.start(); + settableFuture.set(null); + } catch (Throwable e) { + settableFuture.set(e); + } + } + }); + // wait until xdsClientWrapperForServerSds.serverWatchers populated + for (int i = 0; i < 10 && xdsClientWrapperForServerSds.serverWatchers.isEmpty(); i++) { + Thread.sleep(100L); } + return settableFuture; + } + + @Test + public void xdsServerStartAndShutdown() + throws IOException, InterruptedException, TimeoutException, ExecutionException { + buildServer(null, true); + Future future = startServerAsync(); + XdsServerTestHelper.generateListenerUpdate( + listenerWatcher, + port, + CommonTlsContextTestsUtil.buildTestInternalDownstreamTlsContext("CERT1", "VA1"), + null); + verifyServer(future, null); + verifyShutdown(); } + + @Test + public void xdsServerStartAfterListenerUpdate() + throws IOException, InterruptedException, TimeoutException, ExecutionException { + buildServer(null, true); + XdsServerTestHelper.generateListenerUpdate( + listenerWatcher, + port, + CommonTlsContextTestsUtil.buildTestInternalDownstreamTlsContext("CERT1", "VA1"), + null); + xdsServer.start(); + try { + xdsServer.start(); + fail("expected exception"); + } catch (IllegalStateException expected) { + assertThat(expected).hasMessageThat().contains("Already started"); + } + verifyServer(null,null); + verifyShutdown(); + } + + @Test + public void xdsServerStartAndShutdownWithErrorNotifier() + throws IOException, InterruptedException, TimeoutException, ExecutionException { + XdsServerBuilder.ErrorNotifier mockErrorNotifier = mock(XdsServerBuilder.ErrorNotifier.class); + buildServer(mockErrorNotifier, true); + Future future = startServerAsync(); + XdsServerTestHelper.generateListenerUpdate( + listenerWatcher, + port, + CommonTlsContextTestsUtil.buildTestInternalDownstreamTlsContext("CERT1", "VA1"), + null); + verifyServer(future, mockErrorNotifier); + verifyShutdown(); + } + + @Test + public void xdsServer_serverWatcher() + throws IOException, InterruptedException, TimeoutException, ExecutionException { + XdsServerBuilder.ErrorNotifier mockErrorNotifier = mock(XdsServerBuilder.ErrorNotifier.class); + buildServer(mockErrorNotifier, true); + Future future = startServerAsync(); + listenerWatcher.onError(Status.ABORTED); + verify(mockErrorNotifier).onError(Status.ABORTED); + assertThat(xdsClientWrapperForServerSds.serverWatchers).hasSize(1); + assertThat(future.isDone()).isFalse(); + reset(mockErrorNotifier); + listenerWatcher.onResourceDoesNotExist("not found error"); + ArgumentCaptor argCaptor = ArgumentCaptor.forClass(null); + verify(mockErrorNotifier).onError(argCaptor.capture()); + Status captured = argCaptor.getValue(); + assertThat(captured.getCode()).isEqualTo(Status.Code.NOT_FOUND); + assertThat(captured.getDescription()).isEqualTo("not found error"); + assertThat(xdsClientWrapperForServerSds.serverWatchers).hasSize(1); + assertThat(future.isDone()).isFalse(); + reset(mockErrorNotifier); + XdsServerTestHelper.generateListenerUpdate( + listenerWatcher, + port, + CommonTlsContextTestsUtil.buildTestInternalDownstreamTlsContext("CERT1", "VA1"), + null); + verifyServer(future, mockErrorNotifier); + verifyShutdown(); + } + + @Test + public void xdsServer_startError() + throws IOException, InterruptedException, TimeoutException, ExecutionException { + XdsServerBuilder.ErrorNotifier mockErrorNotifier = mock(XdsServerBuilder.ErrorNotifier.class); + buildServer(mockErrorNotifier, true); + Future future = startServerAsync(); + // create port conflict for start to fail + ServerSocket serverSocket = new ServerSocket(port); + XdsServerTestHelper.generateListenerUpdate( + listenerWatcher, + port, + CommonTlsContextTestsUtil.buildTestInternalDownstreamTlsContext("CERT1", "VA1"), + null); + Throwable exception = future.get(5, TimeUnit.SECONDS); + assertThat(exception).isInstanceOf(IOException.class); + assertThat(exception).hasMessageThat().contains("Failed to bind"); + verify(mockErrorNotifier, never()).onError(any(Status.class)); + serverSocket.close(); + } + + @Test + public void xdsServerWithoutMockXdsClient_startError() + throws IOException, InterruptedException, TimeoutException, ExecutionException { + XdsServerBuilder.ErrorNotifier mockErrorNotifier = mock(XdsServerBuilder.ErrorNotifier.class); + buildServer(mockErrorNotifier, false); + try { + xdsServer.start(); + fail("exception expected"); + } catch (IOException expected) { + assertThat(expected) + .hasMessageThat() + .contains("Environment variable GRPC_XDS_BOOTSTRAP not defined"); + } + ArgumentCaptor argCaptor = ArgumentCaptor.forClass(null); + verify(mockErrorNotifier).onError(argCaptor.capture()); + Status captured = argCaptor.getValue(); + assertThat(captured.getCode()).isEqualTo(Status.Code.UNKNOWN); + assertThat(captured.getCause()).isInstanceOf(IOException.class); + assertThat(captured.getCause()) + .hasMessageThat() + .contains("Environment variable GRPC_XDS_BOOTSTRAP not defined"); + } + + @Test + public void xdsServerStartSecondUpdateAndError() + throws IOException, InterruptedException, TimeoutException, ExecutionException { + XdsServerBuilder.ErrorNotifier mockErrorNotifier = mock(XdsServerBuilder.ErrorNotifier.class); + buildServer(mockErrorNotifier, true); + Future future = startServerAsync(); + XdsServerTestHelper.generateListenerUpdate( + listenerWatcher, + port, + CommonTlsContextTestsUtil.buildTestInternalDownstreamTlsContext("CERT1", "VA1"), + null); + XdsServerTestHelper.generateListenerUpdate( + listenerWatcher, + port, + CommonTlsContextTestsUtil.buildTestInternalDownstreamTlsContext("CERT1", "VA1"), + null); + verify(mockErrorNotifier, never()).onError(any(Status.class)); + verifyServer(future, mockErrorNotifier); + listenerWatcher.onError(Status.ABORTED); + verify(mockErrorNotifier, never()).onError(any(Status.class)); + verifyServer(null, mockErrorNotifier); + verifyShutdown(); + } + } diff --git a/xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java b/xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java new file mode 100644 index 00000000000..75e0b8d00ae --- /dev/null +++ b/xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java @@ -0,0 +1,119 @@ +/* + * Copyright 2020 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.xds; + +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; + +import com.google.common.base.Strings; +import io.grpc.xds.internal.sds.XdsServerBuilder; +import java.io.IOException; +import java.net.ServerSocket; +import java.util.ArrayList; +import java.util.Arrays; + +import org.mockito.ArgumentCaptor; + +/** + * Helper methods related to {@link XdsServerBuilder} and related classes. + */ +class XdsServerTestHelper { + + static XdsClient.ListenerWatcher startAndGetWatcher( + XdsClientWrapperForServerSds xdsClientWrapperForServerSds, + XdsClient mockXdsClient, + int port) { + xdsClientWrapperForServerSds.start(mockXdsClient); + ArgumentCaptor listenerWatcherCaptor = ArgumentCaptor.forClass(null); + verify(mockXdsClient).watchListenerData(eq(port), listenerWatcherCaptor.capture()); + return listenerWatcherCaptor.getValue(); + } + + /** + * Creates a {@link XdsClient.ListenerUpdate} with maximum of 2 + * {@link io.grpc.xds.EnvoyServerProtoData.FilterChain} each one with a destination port and an + * optional {@link EnvoyServerProtoData.DownstreamTlsContext}. + * @param registeredWatcher the watcher on which to generate the update + * @param destPort if > 0 to create both the {@link EnvoyServerProtoData.FilterChain} + * @param tlsContext1 if non-null, used to populate the 1st filterChain + * @param tlsContext2 if non-null, used to populate the 2nd filterChain + */ + static void generateListenerUpdate( + XdsClient.ListenerWatcher registeredWatcher, + int destPort, + EnvoyServerProtoData.DownstreamTlsContext tlsContext1, + EnvoyServerProtoData.DownstreamTlsContext tlsContext2) { + EnvoyServerProtoData.Listener listener = + buildTestListener( + "listener1", + "10.1.2.3", + destPort, + destPort, + null, + null, + null, + null, + tlsContext1, + tlsContext2); + XdsClient.ListenerUpdate listenerUpdate = + XdsClient.ListenerUpdate.newBuilder().setListener(listener).build(); + registeredWatcher.onListenerChanged(listenerUpdate); + } + + static int findFreePort() throws IOException { + try (ServerSocket socket = new ServerSocket(0)) { + socket.setReuseAddress(true); + return socket.getLocalPort(); + } + } + + static EnvoyServerProtoData.Listener buildTestListener( + String name, + String address, + int destPort1, + int destPort2, + String addressPrefix11, + String addressPrefix12, + String addressPrefix21, + String addressPrefix22, + EnvoyServerProtoData.DownstreamTlsContext tlsContext1, + EnvoyServerProtoData.DownstreamTlsContext tlsContext2) { + EnvoyServerProtoData.FilterChainMatch filterChainMatch1 = + destPort1 > 0 ? buildFilterChainMatch(destPort1, addressPrefix11, addressPrefix12) : null; + EnvoyServerProtoData.FilterChainMatch filterChainMatch2 = + destPort2 > 0 ? buildFilterChainMatch(destPort2, addressPrefix21, addressPrefix22) : null; + EnvoyServerProtoData.FilterChain filterChain1 = + new EnvoyServerProtoData.FilterChain(filterChainMatch1, tlsContext1); + EnvoyServerProtoData.FilterChain filterChain2 = + new EnvoyServerProtoData.FilterChain(filterChainMatch2, tlsContext2); + EnvoyServerProtoData.Listener listener = + new EnvoyServerProtoData.Listener(name, address, Arrays.asList(filterChain1, filterChain2)); + return listener; + } + + static EnvoyServerProtoData.FilterChainMatch buildFilterChainMatch( + int destPort, String... addressPrefix) { + ArrayList prefixRanges = new ArrayList<>(); + for (String address : addressPrefix) { + if (!Strings.isNullOrEmpty(address)) { + prefixRanges.add(new EnvoyServerProtoData.CidrRange(address, 32)); + } + } + return new EnvoyServerProtoData.FilterChainMatch( + destPort, prefixRanges, Arrays.asList()); + } +} From c88feeffe833b38e4f0a15b9e6073074f02fd951 Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Tue, 8 Sep 2020 18:25:28 -0700 Subject: [PATCH 25/86] xds: always generate xds_routing LB policy as the top-level LB policy in legacy xDS resolver (to be deprecated) (#7401) Now that routing is enabled by default, we should always generate a load balancing config that uses xds_routing as the top-level LB policy. This change is made to the legacy xDS resolver, which will soon be replaced by XdsNameResolver2. But this change allows us to merge split codepath for CDS LB policy. --- .../java/io/grpc/xds/XdsNameResolver.java | 36 +----- .../xds/XdsNameResolverIntegrationTest.java | 109 +++++------------- 2 files changed, 33 insertions(+), 112 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java index e0c03e5f907..a94360848fe 100644 --- a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java +++ b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java @@ -23,7 +23,6 @@ import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; import com.google.gson.Gson; import com.google.re2j.Pattern; import io.grpc.Attributes; @@ -155,35 +154,12 @@ private class ConfigWatcherImpl implements ConfigWatcher { @Override public void onConfigChanged(ConfigUpdate update) { - Map rawLbConfig; - if (update.getRoutes().size() > 1) { - logger.log( - XdsLogLevel.INFO, - "Received config update with {0} routes from xDS client {1}", - update.getRoutes().size(), - xdsClient); - rawLbConfig = generateXdsRoutingRawConfig(update.getRoutes()); - } else { - Route defaultRoute = Iterables.getOnlyElement(update.getRoutes()); - RouteAction action = defaultRoute.getRouteAction(); - String clusterName = defaultRoute.getRouteAction().getCluster(); - if (action.getCluster() != null) { - logger.log( - XdsLogLevel.INFO, - "Received config update from xDS client {0}: cluster_name={1}", - xdsClient, - clusterName); - rawLbConfig = generateCdsRawConfig(clusterName); - } else { - logger.log( - XdsLogLevel.INFO, - "Received config update with one weighted cluster route from xDS client {0}", - xdsClient); - List clusterWeights = defaultRoute.getRouteAction().getWeightedCluster(); - rawLbConfig = generateWeightedTargetRawConfig(clusterWeights); - } - } - + logger.log( + XdsLogLevel.INFO, + "Received config update with {0} routes from xDS client {1}", + update.getRoutes().size(), + xdsClient); + Map rawLbConfig = generateXdsRoutingRawConfig(update.getRoutes()); Map serviceConfig = ImmutableMap.of("loadBalancingConfig", ImmutableList.of(rawLbConfig)); if (logger.isLoggable(XdsLogLevel.INFO)) { diff --git a/xds/src/test/java/io/grpc/xds/XdsNameResolverIntegrationTest.java b/xds/src/test/java/io/grpc/xds/XdsNameResolverIntegrationTest.java index d911b34ec0b..e843fd14a40 100644 --- a/xds/src/test/java/io/grpc/xds/XdsNameResolverIntegrationTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsNameResolverIntegrationTest.java @@ -22,7 +22,6 @@ import static io.grpc.xds.XdsClientTestHelper.buildRouteConfigurationV2; import static io.grpc.xds.XdsClientTestHelper.buildVirtualHostV2; import static io.grpc.xds.XdsNameResolverTest.assertCdsPolicy; -import static io.grpc.xds.XdsNameResolverTest.assertWeightedTargetConfigClusterWeights; import static io.grpc.xds.XdsNameResolverTest.assertWeightedTargetPolicy; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -295,9 +294,15 @@ public void resolve_resourceUpdated() { List> rawLbConfigs = (List>) serviceConfig.get("loadBalancingConfig"); Map lbConfig = Iterables.getOnlyElement(rawLbConfigs); - assertThat(lbConfig.keySet()).containsExactly("cds_experimental"); - Map rawConfigValues = (Map) lbConfig.get("cds_experimental"); - assertThat(rawConfigValues).containsExactly("cluster", "cluster-foo.googleapis.com"); + assertThat(lbConfig.keySet()).containsExactly("xds_routing_experimental"); + Map rawConfigValues = (Map) lbConfig.get("xds_routing_experimental"); + Map> actions = + (Map>) rawConfigValues.get("action"); + List> routes = (List>) rawConfigValues.get("route"); + Map route = Iterables.getOnlyElement(routes); + assertThat(route.keySet()).containsExactly("prefix", "action"); + assertThat((String) route.get("prefix")).isEqualTo(""); + assertCdsPolicy(actions.get(route.get("action")), "cluster-foo.googleapis.com"); // Simulate receiving another LDS response that tells client to do RDS. String routeConfigName = "route-foo.googleapis.com"; @@ -318,33 +323,14 @@ public void resolve_resourceUpdated() { serviceConfig = (Map) result.getServiceConfig().getConfig(); rawLbConfigs = (List>) serviceConfig.get("loadBalancingConfig"); lbConfig = Iterables.getOnlyElement(rawLbConfigs); - assertThat(lbConfig.keySet()).containsExactly("cds_experimental"); - rawConfigValues = (Map) lbConfig.get("cds_experimental"); - assertThat(rawConfigValues).containsExactly("cluster", "cluster-blade.googleapis.com"); - } - - @SuppressWarnings("unchecked") - @Test - public void resolve_cdsLoadBalancing() { - xdsNameResolver.start(mockListener); - assertThat(responseObservers).hasSize(1); - StreamObserver responseObserver = responseObservers.poll(); - - // Simulate receiving an LDS response that contains cluster resolution directly in-line. - String clusterName = "cluster-foo.googleapis.com"; - responseObserver.onNext( - buildLdsResponseForCluster("0", AUTHORITY, clusterName, "0000")); - - verify(mockListener).onResult(resolutionResultCaptor.capture()); - ResolutionResult result = resolutionResultCaptor.getValue(); - assertThat(result.getAddresses()).isEmpty(); - Map serviceConfig = (Map) result.getServiceConfig().getConfig(); - List> rawLbConfigs = - (List>) serviceConfig.get("loadBalancingConfig"); - Map lbConfig = Iterables.getOnlyElement(rawLbConfigs); - assertThat(lbConfig.keySet()).containsExactly("cds_experimental"); - Map rawConfigValues = (Map) lbConfig.get("cds_experimental"); - assertThat(rawConfigValues).containsExactly("cluster", clusterName); + assertThat(lbConfig.keySet()).containsExactly("xds_routing_experimental"); + rawConfigValues = (Map) lbConfig.get("xds_routing_experimental"); + actions = (Map>) rawConfigValues.get("action"); + routes = (List>) rawConfigValues.get("route"); + route = Iterables.getOnlyElement(routes); + assertThat(route.keySet()).containsExactly("prefix", "action"); + assertThat((String) route.get("prefix")).isEqualTo(""); + assertCdsPolicy(actions.get(route.get("action")), "cluster-blade.googleapis.com"); } @Test @@ -447,54 +433,6 @@ public void resolve_xdsRoutingLoadBalancing() { assertCdsPolicy(actions.get(route4.get("action")), "cluster-hello.googleapis.com"); } - @SuppressWarnings("unchecked") - @Test - public void resolve_weightedTargetLoadBalancing() { - xdsNameResolver.start(mockListener); - assertThat(responseObservers).hasSize(1); - StreamObserver responseObserver = responseObservers.poll(); - - // Simulate receiving another LDS response that tells client to do RDS. - String routeConfigName = "route-foo.googleapis.com"; - responseObserver.onNext( - buildLdsResponseForRdsResource("1", AUTHORITY, routeConfigName, "0001")); - - // Client sent an RDS request for resource "route-foo.googleapis.com" (Omitted in this test). - - // Simulate receiving an RDS response that contains the resource "route-foo.googleapis.com" - // with a route resolution for a single weighted cluster route. - Route weightedClustersDefaultRoute = - Route.newBuilder() - .setMatch(RouteMatch.newBuilder().setPrefix("")) - .setRoute(buildWeightedClusterRoute( - ImmutableMap.of( - "cluster-foo.googleapis.com", 20, "cluster-bar.googleapis.com", 80))) - .build(); - List routeConfigs = ImmutableList.of( - Any.pack( - buildRouteConfigurationV2( - routeConfigName, - ImmutableList.of( - buildVirtualHostForRoutes( - AUTHORITY, ImmutableList.of(weightedClustersDefaultRoute)))))); - responseObserver.onNext( - buildDiscoveryResponseV2("0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000")); - - verify(mockListener).onResult(resolutionResultCaptor.capture()); - ResolutionResult result = resolutionResultCaptor.getValue(); - assertThat(result.getAddresses()).isEmpty(); - Map serviceConfig = (Map) result.getServiceConfig().getConfig(); - List> rawLbConfigs = - (List>) serviceConfig.get("loadBalancingConfig"); - Map lbConfig = Iterables.getOnlyElement(rawLbConfigs); - assertThat(lbConfig.keySet()).containsExactly("weighted_target_experimental"); - Map rawConfigValues = (Map) lbConfig.get("weighted_target_experimental"); - assertWeightedTargetConfigClusterWeights( - rawConfigValues, - ImmutableMap.of( - "cluster-foo.googleapis.com", 20, "cluster-bar.googleapis.com", 80)); - } - @Test @SuppressWarnings("unchecked") public void resolve_resourceNewlyAdded() { @@ -511,6 +449,7 @@ public void resolve_resourceNewlyAdded() { verify(mockListener).onResult(resolutionResultCaptor.capture()); ResolutionResult result = resolutionResultCaptor.getValue(); assertThat(result.getAddresses()).isEmpty(); + assertThat((Map) result.getServiceConfig().getConfig()).isEmpty(); // Simulate receiving another LDS response that contains cluster resolution directly in-line. responseObserver.onNext( @@ -524,9 +463,15 @@ public void resolve_resourceNewlyAdded() { List> rawLbConfigs = (List>) serviceConfig.get("loadBalancingConfig"); Map lbConfig = Iterables.getOnlyElement(rawLbConfigs); - assertThat(lbConfig.keySet()).containsExactly("cds_experimental"); - Map rawConfigValues = (Map) lbConfig.get("cds_experimental"); - assertThat(rawConfigValues).containsExactly("cluster", "cluster-foo.googleapis.com"); + assertThat(lbConfig.keySet()).containsExactly("xds_routing_experimental"); + Map rawConfigValues = (Map) lbConfig.get("xds_routing_experimental"); + Map> actions = + (Map>) rawConfigValues.get("action"); + List> routes = (List>) rawConfigValues.get("route"); + Map route = Iterables.getOnlyElement(routes); + assertThat(route.keySet()).containsExactly("prefix", "action"); + assertThat((String) route.get("prefix")).isEqualTo(""); + assertCdsPolicy(actions.get(route.get("action")), "cluster-foo.googleapis.com"); } /** From 431df72d2bd9cd5ab626777f819faed581f07762 Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Wed, 9 Sep 2020 12:14:03 -0700 Subject: [PATCH 26/86] xds: eliminate cluster name change logic in CDS LB policy and reimplement tests (roll forward #7356) (#7395) Eliminated the logic in CDS LB policy that handles CDS config change (aka, cluster name change). A CDS LB policy should be used for a single cluster, the cluster should be effectively final. If the upstream LB policy needs to change routing cluster, it should create separate CDS LB policies, one for each cluster. This change also reimplemented the unit tests for CDS LB policy. --- .../java/io/grpc/xds/CdsLoadBalancer.java | 199 ++-- .../java/io/grpc/xds/CdsLoadBalancerTest.java | 937 ++++++++---------- 2 files changed, 475 insertions(+), 661 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java index a97846fa54c..5dfe6d5444d 100644 --- a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java @@ -32,7 +32,6 @@ import io.grpc.internal.ObjectPool; import io.grpc.internal.ServiceConfigUtil.PolicySelection; import io.grpc.util.ForwardingLoadBalancerHelper; -import io.grpc.util.GracefulSwitchLoadBalancer; import io.grpc.xds.CdsLoadBalancerProvider.CdsConfig; import io.grpc.xds.EdsLoadBalancerProvider.EdsConfig; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; @@ -45,31 +44,24 @@ import io.grpc.xds.internal.sds.TlsContextManagerImpl; import java.util.ArrayList; import java.util.List; -import java.util.Objects; -import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nullable; /** - * Load balancer for cds_experimental LB policy. + * Load balancer for cds_experimental LB policy. One instance per cluster. */ -public final class CdsLoadBalancer extends LoadBalancer { +final class CdsLoadBalancer extends LoadBalancer { private final XdsLogger logger; + private final Helper helper; private final LoadBalancerRegistry lbRegistry; - private final GracefulSwitchLoadBalancer switchingLoadBalancer; private final TlsContextManager tlsContextManager; // TODO(sanjaypujare): remove once xds security is released private boolean enableXdsSecurity; private static final String XDS_SECURITY_ENV_VAR = "GRPC_XDS_EXPERIMENTAL_SECURITY_SUPPORT"; - - // The following fields become non-null once handleResolvedAddresses() successfully. - - // Most recent cluster name. - @Nullable private String clusterName; - @Nullable private ObjectPool xdsClientPool; - @Nullable private XdsClient xdsClient; + private CdsLbState cdsLbState; + private ResolvedAddresses resolvedAddresses; CdsLoadBalancer(Helper helper) { this(helper, LoadBalancerRegistry.getDefaultRegistry(), TlsContextManagerImpl.getInstance()); @@ -78,9 +70,8 @@ public final class CdsLoadBalancer extends LoadBalancer { @VisibleForTesting CdsLoadBalancer(Helper helper, LoadBalancerRegistry lbRegistry, TlsContextManager tlsContextManager) { - checkNotNull(helper, "helper"); + this.helper = checkNotNull(helper, "helper"); this.lbRegistry = lbRegistry; - this.switchingLoadBalancer = new GracefulSwitchLoadBalancer(helper); this.tlsContextManager = tlsContextManager; logger = XdsLogger.withLogId(InternalLogId.allocate("cds-lb", helper.getAuthority())); logger.log(XdsLogLevel.INFO, "Created"); @@ -88,33 +79,32 @@ public final class CdsLoadBalancer extends LoadBalancer { @Override public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { - logger.log(XdsLogLevel.DEBUG, "Received resolution result: {0}", resolvedAddresses); - if (xdsClientPool == null) { - xdsClientPool = resolvedAddresses.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL); - checkNotNull(xdsClientPool, "missing xDS client pool"); - xdsClient = xdsClientPool.getObject(); + if (clusterName != null) { + return; } - + logger.log(XdsLogLevel.DEBUG, "Received resolution result: {0}", resolvedAddresses); + this.resolvedAddresses = resolvedAddresses; + xdsClientPool = resolvedAddresses.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL); + checkNotNull(xdsClientPool, "missing xDS client pool"); + xdsClient = xdsClientPool.getObject(); Object lbConfig = resolvedAddresses.getLoadBalancingPolicyConfig(); checkNotNull(lbConfig, "missing CDS lb config"); CdsConfig newCdsConfig = (CdsConfig) lbConfig; logger.log( XdsLogLevel.INFO, "Received CDS lb config: cluster={0}", newCdsConfig.name); - - // If cluster is changed, do a graceful switch. - if (!newCdsConfig.name.equals(clusterName)) { - LoadBalancer.Factory clusterBalancerFactory = new ClusterBalancerFactory(newCdsConfig.name); - switchingLoadBalancer.switchTo(clusterBalancerFactory); - } - switchingLoadBalancer.handleResolvedAddresses(resolvedAddresses); clusterName = newCdsConfig.name; + cdsLbState = new CdsLbState(); } @Override public void handleNameResolutionError(Status error) { logger.log(XdsLogLevel.WARNING, "Received name resolution error: {0}", error); - switchingLoadBalancer.handleNameResolutionError(error); + if (cdsLbState != null) { + cdsLbState.propagateError(error); + } else { + helper.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(error)); + } } @Override @@ -125,7 +115,9 @@ public boolean canHandleEmptyAddressListFromNameResolution() { @Override public void shutdown() { logger.log(XdsLogLevel.INFO, "Shutdown"); - switchingLoadBalancer.shutdown(); + if (cdsLbState != null) { + cdsLbState.shutdown(); + } if (xdsClientPool != null) { xdsClientPool.returnObject(xdsClient); } @@ -142,107 +134,25 @@ void setXdsSecurity(boolean enable) { enableXdsSecurity = enable; } - /** - * A load balancer factory that provides a load balancer for a given cluster. - */ - private final class ClusterBalancerFactory extends LoadBalancer.Factory { - - final String clusterName; - - ClusterBalancerFactory(String clusterName) { - this.clusterName = clusterName; - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof ClusterBalancerFactory)) { - return false; - } - ClusterBalancerFactory that = (ClusterBalancerFactory) o; - return clusterName.equals(that.clusterName); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), clusterName); - } - - @Override - public LoadBalancer newLoadBalancer(final Helper helper) { - return new LoadBalancer() { - // Becomes non-null once handleResolvedAddresses() successfully. - // Assigned at most once. - @Nullable - ClusterWatcherImpl clusterWatcher; - - @Override - public void handleNameResolutionError(Status error) { - if (clusterWatcher == null || clusterWatcher.edsBalancer == null) { - // Go into TRANSIENT_FAILURE if we have not yet received any cluster resource. - // Otherwise, we keep running with the data we had previously. - helper.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(error)); - } - } - - @Override - public boolean canHandleEmptyAddressListFromNameResolution() { - return true; - } - - @Override - public void shutdown() { - if (clusterWatcher != null) { - if (clusterWatcher.edsBalancer != null) { - clusterWatcher.edsBalancer.shutdown(); - } - xdsClient.cancelClusterDataWatch(clusterName, clusterWatcher); - logger.log( - XdsLogLevel.INFO, - "Cancelled cluster watcher on {0} with xDS client {1}", - clusterName, xdsClient); - } - } - - @Override - public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { - if (clusterWatcher == null) { - clusterWatcher = new ClusterWatcherImpl(helper, resolvedAddresses); - logger.log( - XdsLogLevel.INFO, - "Start cluster watcher on {0} with xDS client {1}", - clusterName, xdsClient); - xdsClient.watchClusterData(clusterName, clusterWatcher); - } - } - }; - } - } - - private static final class EdsLoadBalancingHelper extends ForwardingLoadBalancerHelper { - private final Helper delegate; - private final AtomicReference sslContextProviderSupplier; - - EdsLoadBalancingHelper(Helper helper, - AtomicReference sslContextProviderSupplier) { - this.delegate = helper; - this.sslContextProviderSupplier = sslContextProviderSupplier; - } + private final class ChannelSecurityLbHelper extends ForwardingLoadBalancerHelper { + @Nullable + private SslContextProviderSupplier sslContextProviderSupplier; @Override public Subchannel createSubchannel(CreateSubchannelArgs createSubchannelArgs) { - if (sslContextProviderSupplier.get() != null) { + if (sslContextProviderSupplier != null) { createSubchannelArgs = createSubchannelArgs .toBuilder() .setAddresses( addSslContextProviderSupplier(createSubchannelArgs.getAddresses(), - sslContextProviderSupplier.get())) + sslContextProviderSupplier)) .build(); } - return delegate.createSubchannel(createSubchannelArgs); + return delegate().createSubchannel(createSubchannelArgs); } - private static List addSslContextProviderSupplier( + private List addSslContextProviderSupplier( List addresses, SslContextProviderSupplier supplier) { if (supplier == null || addresses == null) { @@ -253,10 +163,9 @@ private static List addSslContextProviderSupplier( EquivalentAddressGroup eagCopy = new EquivalentAddressGroup(eag.getAddresses(), eag.getAttributes() - .toBuilder() - .set(XdsAttributes.ATTR_SSL_CONTEXT_PROVIDER_SUPPLIER, supplier) - .build() - ); + .toBuilder() + .set(XdsAttributes.ATTR_SSL_CONTEXT_PROVIDER_SUPPLIER, supplier) + .build()); copyList.add(eagCopy); } return copyList; @@ -264,22 +173,19 @@ private static List addSslContextProviderSupplier( @Override protected Helper delegate() { - return delegate; + return helper; } } - private final class ClusterWatcherImpl implements ClusterWatcher { - - final EdsLoadBalancingHelper helper; - final ResolvedAddresses resolvedAddresses; - + private final class CdsLbState implements ClusterWatcher { + private final ChannelSecurityLbHelper lbHelper = new ChannelSecurityLbHelper(); @Nullable LoadBalancer edsBalancer; - ClusterWatcherImpl(Helper helper, ResolvedAddresses resolvedAddresses) { - this.helper = new EdsLoadBalancingHelper(helper, - new AtomicReference()); - this.resolvedAddresses = resolvedAddresses; + private CdsLbState() { + xdsClient.watchClusterData(clusterName, this); + logger.log(XdsLogLevel.INFO, + "Started watcher for cluster {0} with xDS client {1}", clusterName, xdsClient); } @Override @@ -308,7 +214,7 @@ public void onClusterChanged(ClusterUpdate newUpdate) { updateSslContextProviderSupplier(newUpdate.getUpstreamTlsContext()); } if (edsBalancer == null) { - edsBalancer = lbRegistry.getProvider(EDS_POLICY_NAME).newLoadBalancer(helper); + edsBalancer = lbRegistry.getProvider(EDS_POLICY_NAME).newLoadBalancer(lbHelper); } edsBalancer.handleResolvedAddresses( resolvedAddresses.toBuilder().setLoadBalancingPolicyConfig(edsConfig).build()); @@ -317,7 +223,7 @@ public void onClusterChanged(ClusterUpdate newUpdate) { /** For new UpstreamTlsContext value, release old SslContextProvider. */ private void updateSslContextProviderSupplier(UpstreamTlsContext newUpstreamTlsContext) { SslContextProviderSupplier oldSslContextProviderSupplier = - helper.sslContextProviderSupplier.get(); + lbHelper.sslContextProviderSupplier; if (oldSslContextProviderSupplier != null) { UpstreamTlsContext oldUpstreamTlsContext = oldSslContextProviderSupplier.getUpstreamTlsContext(); @@ -328,11 +234,10 @@ private void updateSslContextProviderSupplier(UpstreamTlsContext newUpstreamTlsC oldSslContextProviderSupplier.close(); } if (newUpstreamTlsContext != null) { - SslContextProviderSupplier newSslContextProviderSupplier = + lbHelper.sslContextProviderSupplier = new SslContextProviderSupplier(newUpstreamTlsContext, tlsContextManager); - helper.sslContextProviderSupplier.set(newSslContextProviderSupplier); } else { - helper.sslContextProviderSupplier.set(null); + lbHelper.sslContextProviderSupplier = null; } } @@ -361,6 +266,22 @@ public void onError(Status error) { helper.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(error)); } } - } + void shutdown() { + xdsClient.cancelClusterDataWatch(clusterName, this); + logger.log(XdsLogLevel.INFO, + "Cancelled watcher for cluster {0} with xDS client {1}", clusterName, xdsClient); + if (edsBalancer != null) { + edsBalancer.shutdown(); + } + } + + void propagateError(Status error) { + if (edsBalancer != null) { + edsBalancer.handleNameResolutionError(error); + } else { + helper.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(error)); + } + } + } } diff --git a/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java index 77a104262d4..4c0a1d4b324 100644 --- a/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java @@ -17,27 +17,10 @@ package io.grpc.xds; import static com.google.common.truth.Truth.assertThat; -import static io.grpc.ConnectivityState.CONNECTING; -import static io.grpc.ConnectivityState.READY; -import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; -import static io.grpc.xds.XdsLbPolicies.EDS_POLICY_NAME; -import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.BAD_CLIENT_KEY_FILE; -import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.BAD_CLIENT_PEM_FILE; -import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.CA_PEM_FILE; -import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.CLIENT_KEY_FILE; -import static io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.CLIENT_PEM_FILE; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.same; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; -import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.collect.Iterables; import io.grpc.Attributes; import io.grpc.ConnectivityState; import io.grpc.EquivalentAddressGroup; @@ -47,40 +30,35 @@ import io.grpc.LoadBalancer.PickResult; import io.grpc.LoadBalancer.PickSubchannelArgs; import io.grpc.LoadBalancer.ResolvedAddresses; +import io.grpc.LoadBalancer.Subchannel; import io.grpc.LoadBalancer.SubchannelPicker; import io.grpc.LoadBalancerProvider; import io.grpc.LoadBalancerRegistry; -import io.grpc.NameResolver.ConfigOrError; +import io.grpc.ManagedChannel; +import io.grpc.NameResolver; import io.grpc.Status; import io.grpc.Status.Code; import io.grpc.SynchronizationContext; -import io.grpc.internal.FakeClock; -import io.grpc.internal.ServiceConfigUtil.PolicySelection; +import io.grpc.internal.ObjectPool; import io.grpc.xds.CdsLoadBalancerProvider.CdsConfig; import io.grpc.xds.EdsLoadBalancerProvider.EdsConfig; +import io.grpc.xds.EnvoyServerProtoData.DownstreamTlsContext; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; -import io.grpc.xds.XdsClient.ClusterUpdate; -import io.grpc.xds.XdsClient.ClusterWatcher; -import io.grpc.xds.XdsClient.RefCountedXdsClientObjectPool; -import io.grpc.xds.XdsClient.XdsClientFactory; import io.grpc.xds.internal.sds.CommonTlsContextTestsUtil; import io.grpc.xds.internal.sds.SslContextProvider; import io.grpc.xds.internal.sds.SslContextProviderSupplier; import io.grpc.xds.internal.sds.TlsContextManager; -import io.netty.handler.ssl.SslContext; -import java.net.InetSocketAddress; -import java.util.ArrayDeque; +import java.net.SocketAddress; import java.util.ArrayList; -import java.util.Deque; -import java.util.HashMap; +import java.util.Collections; import java.util.List; -import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -import org.mockito.ArgumentCaptor; -import org.mockito.Mock; import org.mockito.MockitoAnnotations; /** @@ -88,44 +66,326 @@ */ @RunWith(JUnit4.class) public class CdsLoadBalancerTest { - - private final RefCountedXdsClientObjectPool xdsClientPool = new RefCountedXdsClientObjectPool( - new XdsClientFactory() { + private static final String AUTHORITY = "api.google.com"; + private static final String CLUSTER = "cluster-foo.googleapis.com"; + private final SynchronizationContext syncContext = new SynchronizationContext( + new Thread.UncaughtExceptionHandler() { @Override - XdsClient createXdsClient() { - xdsClient = mock(XdsClient.class); - return xdsClient; + public void uncaughtException(Thread t, Throwable e) { + throw new AssertionError(e); } + }); + private final List childBalancers = new ArrayList<>(); + private final FakeXdsClient xdsClient = new FakeXdsClient(); + private final TlsContextManager tlsContextManager = new FakeTlsContextManager(); + private LoadBalancer.Helper helper = new FakeLbHelper(); + private int xdsClientRefs; + private ConnectivityState currentState; + private SubchannelPicker currentPicker; + private CdsLoadBalancer loadBalancer; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + + LoadBalancerRegistry registry = new LoadBalancerRegistry(); + registry.register(new FakeLoadBalancerProvider(XdsLbPolicies.EDS_POLICY_NAME)); + registry.register(new FakeLoadBalancerProvider("round_robin")); + ObjectPool xdsClientPool = new ObjectPool() { + @Override + public XdsClient getObject() { + xdsClientRefs++; + return xdsClient; } - ); - private final LoadBalancerRegistry lbRegistry = new LoadBalancerRegistry(); - private final LoadBalancerProvider fakeEdsLoadBlancerProvider = new LoadBalancerProvider() { + @Override + public XdsClient returnObject(Object object) { + assertThat(xdsClientRefs).isGreaterThan(0); + xdsClientRefs--; + return null; + } + }; + loadBalancer = new CdsLoadBalancer(helper, registry, tlsContextManager); + loadBalancer.handleResolvedAddresses( + ResolvedAddresses.newBuilder() + .setAddresses(Collections.emptyList()) + .setLoadBalancingPolicyConfig(new CdsConfig(CLUSTER)) + .setAttributes( + Attributes.newBuilder().set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool).build()) + .build()); + assertThat(xdsClient.watcher).isNotNull(); + } + + @After + public void tearDown() { + loadBalancer.shutdown(); + assertThat(xdsClient.watcher).isNull(); + assertThat(xdsClientRefs).isEqualTo(0); + } + + + @Test + public void receiveFirstClusterResourceInfo() { + xdsClient.deliverClusterInfo(null, null); + assertThat(childBalancers).hasSize(1); + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); + assertThat(childBalancer.name).isEqualTo(XdsLbPolicies.EDS_POLICY_NAME); + assertThat(childBalancer.config).isNotNull(); + EdsConfig edsConfig = (EdsConfig) childBalancer.config; + assertThat(edsConfig.clusterName).isEqualTo(CLUSTER); + assertThat(edsConfig.edsServiceName).isNull(); + assertThat(edsConfig.lrsServerName).isNull(); + assertThat(edsConfig.endpointPickingPolicy.getProvider().getPolicyName()) + .isEqualTo("round_robin"); + } + + @Test + public void clusterResourceNeverExist() { + xdsClient.deliverResourceNotFound(); + assertThat(childBalancers).isEmpty(); + assertThat(currentState).isEqualTo(ConnectivityState.TRANSIENT_FAILURE); + PickResult result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); + assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAVAILABLE); + assertThat(result.getStatus().getDescription()) + .isEqualTo("Resource " + CLUSTER + " is unavailable"); + } + + @Test + public void clusterResourceRemoved() { + xdsClient.deliverClusterInfo(null, null); + assertThat(childBalancers).hasSize(1); + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); + assertThat(childBalancer.shutdown).isFalse(); + + xdsClient.deliverResourceNotFound(); + assertThat(childBalancer.shutdown).isTrue(); + assertThat(currentState).isEqualTo(ConnectivityState.TRANSIENT_FAILURE); + PickResult result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); + assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAVAILABLE); + assertThat(result.getStatus().getDescription()) + .isEqualTo("Resource " + CLUSTER + " is unavailable"); + } + + @Test + public void clusterResourceUpdated() { + xdsClient.deliverClusterInfo(null, null); + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); + EdsConfig edsConfig = (EdsConfig) childBalancer.config; + assertThat(edsConfig.clusterName).isEqualTo(CLUSTER); + assertThat(edsConfig.edsServiceName).isNull(); + assertThat(edsConfig.lrsServerName).isNull(); + assertThat(edsConfig.endpointPickingPolicy.getProvider().getPolicyName()) + .isEqualTo("round_robin"); + + String edsService = "service-bar.googleapis.com"; + String loadReportServer = "lrs-server.googleapis.com"; + xdsClient.deliverClusterInfo(edsService, loadReportServer); + assertThat(childBalancers).containsExactly(childBalancer); + edsConfig = (EdsConfig) childBalancer.config; + assertThat(edsConfig.clusterName).isEqualTo(CLUSTER); + assertThat(edsConfig.edsServiceName).isEqualTo(edsService); + assertThat(edsConfig.lrsServerName).isEqualTo(loadReportServer); + assertThat(edsConfig.endpointPickingPolicy.getProvider().getPolicyName()) + .isEqualTo("round_robin"); + } + + @Test + public void receiveClusterResourceInfoWithUpstreamTlsContext() { + loadBalancer.setXdsSecurity(true); + UpstreamTlsContext upstreamTlsContext = + CommonTlsContextTestsUtil.buildUpstreamTlsContextFromFilenames( + CommonTlsContextTestsUtil.CLIENT_KEY_FILE, + CommonTlsContextTestsUtil.CLIENT_PEM_FILE, + CommonTlsContextTestsUtil.CA_PEM_FILE); + xdsClient.deliverClusterInfo(null, null, upstreamTlsContext); + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); + List addresses = createEndpointAddresses(2); + CreateSubchannelArgs args = + CreateSubchannelArgs.newBuilder() + .setAddresses(addresses) + .build(); + Subchannel subchannel = childBalancer.helper.createSubchannel(args); + for (EquivalentAddressGroup eag : subchannel.getAllAddresses()) { + SslContextProviderSupplier supplier = + eag.getAttributes().get(XdsAttributes.ATTR_SSL_CONTEXT_PROVIDER_SUPPLIER); + assertThat(supplier.getUpstreamTlsContext()).isEqualTo(upstreamTlsContext); + } + + xdsClient.deliverClusterInfo(null, null); + subchannel = childBalancer.helper.createSubchannel(args); + for (EquivalentAddressGroup eag : subchannel.getAllAddresses()) { + assertThat(eag.getAttributes().get(XdsAttributes.ATTR_SSL_CONTEXT_PROVIDER_SUPPLIER)) + .isNull(); + } + + upstreamTlsContext = + CommonTlsContextTestsUtil.buildUpstreamTlsContextFromFilenames( + CommonTlsContextTestsUtil.BAD_CLIENT_KEY_FILE, + CommonTlsContextTestsUtil.BAD_CLIENT_PEM_FILE, + CommonTlsContextTestsUtil.CA_PEM_FILE); + xdsClient.deliverClusterInfo(null, null, upstreamTlsContext); + subchannel = childBalancer.helper.createSubchannel(args); + for (EquivalentAddressGroup eag : subchannel.getAllAddresses()) { + SslContextProviderSupplier supplier = + eag.getAttributes().get(XdsAttributes.ATTR_SSL_CONTEXT_PROVIDER_SUPPLIER); + assertThat(supplier.getUpstreamTlsContext()).isEqualTo(upstreamTlsContext); + } + } + + @Test + public void subchannelStatePropagateFromDownstreamToUpstream() { + xdsClient.deliverClusterInfo(null, null); + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); + List addresses = createEndpointAddresses(2); + CreateSubchannelArgs args = + CreateSubchannelArgs.newBuilder() + .setAddresses(addresses) + .build(); + Subchannel subchannel = childBalancer.helper.createSubchannel(args); + childBalancer.deliverSubchannelState(subchannel, ConnectivityState.READY); + assertThat(currentState).isEqualTo(ConnectivityState.READY); + assertThat(currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)).getSubchannel()) + .isSameInstanceAs(subchannel); + } + + @Test + public void clusterDiscoveryError_beforeChildPolicyInstantiated_propagateToUpstream() { + xdsClient.deliverError(Status.UNAUTHENTICATED.withDescription("permission denied")); + assertThat(currentState).isEqualTo(ConnectivityState.TRANSIENT_FAILURE); + PickResult result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); + assertThat(result.getStatus().isOk()).isFalse(); + assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAUTHENTICATED); + assertThat(result.getStatus().getDescription()).isEqualTo("permission denied"); + } + + @Test + public void clusterDiscoveryError_afterChildPolicyInstantiated_keepUsingCurrentCluster() { + xdsClient.deliverClusterInfo(null, null); + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); + xdsClient.deliverError(Status.UNAVAILABLE.withDescription("unreachable")); + assertThat(currentState).isNull(); + assertThat(currentPicker).isNull(); + assertThat(childBalancer.shutdown).isFalse(); + } + + @Test + public void nameResolutionError_beforeChildPolicyInstantiated_returnErrorPickerToUpstream() { + loadBalancer.handleNameResolutionError( + Status.UNIMPLEMENTED.withDescription("not found")); + assertThat(currentState).isEqualTo(ConnectivityState.TRANSIENT_FAILURE); + PickResult result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); + assertThat(result.getStatus().isOk()).isFalse(); + assertThat(result.getStatus().getCode()).isEqualTo(Code.UNIMPLEMENTED); + assertThat(result.getStatus().getDescription()).isEqualTo("not found"); + } + + @Test + public void nameResolutionError_afterChildPolicyInstantiated_propagateToDownstream() { + xdsClient.deliverClusterInfo(null, null); + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); + loadBalancer.handleNameResolutionError( + Status.UNAVAILABLE.withDescription("cannot reach server")); + assertThat(childBalancer.upstreamError.getCode()).isEqualTo(Code.UNAVAILABLE); + assertThat(childBalancer.upstreamError.getDescription()) + .isEqualTo("cannot reach server"); + } + + private static List createEndpointAddresses(int n) { + List list = new ArrayList<>(); + for (int i = 0; i < n; i++) { + list.add(new EquivalentAddressGroup(mock(SocketAddress.class))); + } + return list; + } + + private final class FakeXdsClient extends XdsClient { + private ClusterWatcher watcher; + @Override - public boolean isAvailable() { - return true; + void watchClusterData(String clusterName, ClusterWatcher watcher) { + assertThat(clusterName).isEqualTo(CLUSTER); + this.watcher = watcher; } @Override - public int getPriority() { - return 5; + void cancelClusterDataWatch(String clusterName, ClusterWatcher watcher) { + assertThat(clusterName).isEqualTo(CLUSTER); + assertThat(watcher).isSameInstanceAs(this.watcher); + this.watcher = null; } @Override - public String getPolicyName() { - return EDS_POLICY_NAME; + void shutdown() { + // no-op + } + + void deliverClusterInfo( + @Nullable final String edsServiceName, @Nullable final String lrsServerName) { + syncContext.execute(new Runnable() { + @Override + public void run() { + watcher.onClusterChanged( + ClusterUpdate.newBuilder() + .setClusterName(CLUSTER) + .setEdsServiceName(edsServiceName) + .setLbPolicy("round_robin") // only supported policy + .setLrsServerName(lrsServerName) + .build()); + } + }); + } + + void deliverClusterInfo( + @Nullable final String edsServiceName, @Nullable final String lrsServerName, + final UpstreamTlsContext tlsContext) { + syncContext.execute(new Runnable() { + @Override + public void run() { + watcher.onClusterChanged( + ClusterUpdate.newBuilder() + .setClusterName(CLUSTER) + .setEdsServiceName(edsServiceName) + .setLbPolicy("round_robin") // only supported policy + .setLrsServerName(lrsServerName) + .setUpstreamTlsContext(tlsContext) + .build()); + } + }); + } + + void deliverResourceNotFound() { + syncContext.execute(new Runnable() { + @Override + public void run() { + watcher.onResourceDoesNotExist(CLUSTER); + } + }); + } + + void deliverError(final Status error) { + syncContext.execute(new Runnable() { + @Override + public void run() { + watcher.onError(error); + } + }); + } + } + + private final class FakeLoadBalancerProvider extends LoadBalancerProvider { + private final String policyName; + + FakeLoadBalancerProvider(String policyName) { + this.policyName = policyName; } @Override public LoadBalancer newLoadBalancer(Helper helper) { - edsLbHelpers.add(helper); - LoadBalancer edsLoadBalancer = mock(LoadBalancer.class); - edsLoadBalancers.add(edsLoadBalancer); - return edsLoadBalancer; + FakeLoadBalancer balancer = new FakeLoadBalancer(policyName, helper); + childBalancers.add(balancer); + return balancer; } - }; - private final LoadBalancerProvider fakeRoundRobinLbProvider = new LoadBalancerProvider() { @Override public boolean isAvailable() { return true; @@ -133,505 +393,138 @@ public boolean isAvailable() { @Override public int getPriority() { - return 5; + return 0; // doesn't matter } @Override public String getPolicyName() { - return "round_robin"; + return policyName; + } + } + + private final class FakeLoadBalancer extends LoadBalancer { + private final String name; + private final Helper helper; + private Object config; + private Status upstreamError; + private boolean shutdown; + + FakeLoadBalancer(String name, Helper helper) { + this.name = name; + this.helper = helper; } @Override - public LoadBalancer newLoadBalancer(Helper helper) { - return mock(LoadBalancer.class); + public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { + config = resolvedAddresses.getLoadBalancingPolicyConfig(); } @Override - public ConfigOrError parseLoadBalancingPolicyConfig( - Map rawLoadBalancingPolicyConfig) { - return ConfigOrError.fromConfig("fake round robin config"); + public void handleNameResolutionError(Status error) { + upstreamError = error; } - }; - private final SynchronizationContext syncContext = new SynchronizationContext( - new Thread.UncaughtExceptionHandler() { + @Override + public void shutdown() { + shutdown = true; + childBalancers.remove(this); + } + + void deliverSubchannelState(final Subchannel subchannel, ConnectivityState state) { + SubchannelPicker picker = new SubchannelPicker() { @Override - public void uncaughtException(Thread t, Throwable e) { - throw new AssertionError(e); + public PickResult pickSubchannel(PickSubchannelArgs args) { + return PickResult.withSubchannel(subchannel); } - }); - - private final FakeClock fakeClock = new FakeClock(); - private final Deque edsLoadBalancers = new ArrayDeque<>(); - private final Deque edsLbHelpers = new ArrayDeque<>(); - - @Mock - private Helper helper; - - private LoadBalancer cdsLoadBalancer; - private XdsClient xdsClient; - - @Mock - private TlsContextManager mockTlsContextManager; - - @Before - public void setUp() { - MockitoAnnotations.initMocks(this); - - doReturn(syncContext).when(helper).getSynchronizationContext(); - doReturn(fakeClock.getScheduledExecutorService()).when(helper).getScheduledExecutorService(); - lbRegistry.register(fakeEdsLoadBlancerProvider); - lbRegistry.register(fakeRoundRobinLbProvider); - cdsLoadBalancer = new CdsLoadBalancer(helper, lbRegistry, mockTlsContextManager); + }; + helper.updateBalancingState(state, picker); + } } - @Test - public void canHandleEmptyAddressListFromNameResolution() { - assertThat(cdsLoadBalancer.canHandleEmptyAddressListFromNameResolution()).isTrue(); - } + private final class FakeLbHelper extends LoadBalancer.Helper { - @Test - public void handleResolutionErrorBeforeOrAfterCdsWorking() { - ResolvedAddresses resolvedAddresses1 = ResolvedAddresses.newBuilder() - .setAddresses(ImmutableList.of()) - .setAttributes(Attributes.newBuilder() - .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) - .build()) - .setLoadBalancingPolicyConfig(new CdsConfig("foo.googleapis.com")) - .build(); - cdsLoadBalancer.handleResolvedAddresses(resolvedAddresses1); - ArgumentCaptor clusterWatcherCaptor1 = ArgumentCaptor.forClass(null); - verify(xdsClient).watchClusterData(eq("foo.googleapis.com"), clusterWatcherCaptor1.capture()); - ClusterWatcher clusterWatcher1 = clusterWatcherCaptor1.getValue(); - - // handleResolutionError() before receiving any CDS response. - cdsLoadBalancer.handleNameResolutionError(Status.DATA_LOSS.withDescription("fake status")); - verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); - - // CDS response received. - clusterWatcher1.onClusterChanged( - ClusterUpdate.newBuilder() - .setClusterName("foo.googleapis.com") - .setEdsServiceName("edsServiceFoo.googleapis.com") - .setLbPolicy("round_robin") - .build()); - verify(helper).updateBalancingState(eq(CONNECTING), any(SubchannelPicker.class)); + @Override + public void updateBalancingState( + @Nonnull ConnectivityState newState, @Nonnull SubchannelPicker newPicker) { + currentState = newState; + currentPicker = newPicker; + } - // handleResolutionError() after receiving CDS response. - cdsLoadBalancer.handleNameResolutionError(Status.DATA_LOSS.withDescription("fake status")); - // No more TRANSIENT_FAILURE. - verify(helper, times(1)).updateBalancingState( - eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); - } + @Override + public Subchannel createSubchannel(CreateSubchannelArgs args) { + return new FakeSubchannel(args.getAddresses()); + } - @Test - public void handleCdsConfigUpdate() { - assertThat(xdsClient).isNull(); - ResolvedAddresses resolvedAddresses1 = ResolvedAddresses.newBuilder() - .setAddresses(ImmutableList.of()) - .setAttributes(Attributes.newBuilder() - .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) - .build()) - .setLoadBalancingPolicyConfig(new CdsConfig("foo.googleapis.com")) - .build(); - cdsLoadBalancer.handleResolvedAddresses(resolvedAddresses1); - - ArgumentCaptor clusterWatcherCaptor1 = ArgumentCaptor.forClass(null); - verify(xdsClient).watchClusterData(eq("foo.googleapis.com"), clusterWatcherCaptor1.capture()); - - ClusterWatcher clusterWatcher1 = clusterWatcherCaptor1.getValue(); - clusterWatcher1.onClusterChanged( - ClusterUpdate.newBuilder() - .setClusterName("foo.googleapis.com") - .setEdsServiceName("edsServiceFoo.googleapis.com") - .setLbPolicy("round_robin") - .build()); + @Override + public ManagedChannel createOobChannel(EquivalentAddressGroup eag, String authority) { + throw new UnsupportedOperationException("should not be called"); + } - assertThat(edsLbHelpers).hasSize(1); - assertThat(edsLoadBalancers).hasSize(1); - Helper edsLbHelper1 = edsLbHelpers.poll(); - LoadBalancer edsLoadBalancer1 = edsLoadBalancers.poll(); - ArgumentCaptor resolvedAddressesCaptor1 = ArgumentCaptor.forClass(null); - verify(edsLoadBalancer1).handleResolvedAddresses(resolvedAddressesCaptor1.capture()); - PolicySelection roundRobinPolicy = new PolicySelection( - fakeRoundRobinLbProvider, new HashMap(), "fake round robin config"); - EdsConfig expectedEdsConfig = new EdsConfig( - "foo.googleapis.com", - "edsServiceFoo.googleapis.com", - null, - roundRobinPolicy); - ResolvedAddresses resolvedAddressesFoo = resolvedAddressesCaptor1.getValue(); - assertThat(resolvedAddressesFoo.getLoadBalancingPolicyConfig()).isEqualTo(expectedEdsConfig); - assertThat(resolvedAddressesFoo.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL)) - .isSameInstanceAs(xdsClientPool); - - SubchannelPicker picker1 = mock(SubchannelPicker.class); - edsLbHelper1.updateBalancingState(ConnectivityState.READY, picker1); - verify(helper).updateBalancingState(ConnectivityState.READY, picker1); - - ResolvedAddresses resolvedAddresses2 = ResolvedAddresses.newBuilder() - .setAddresses(ImmutableList.of()) - .setAttributes(Attributes.newBuilder() - .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) - .build()) - .setLoadBalancingPolicyConfig(new CdsConfig("bar.googleapis.com")) - .build(); - cdsLoadBalancer.handleResolvedAddresses(resolvedAddresses2); - - ArgumentCaptor clusterWatcherCaptor2 = ArgumentCaptor.forClass(null); - verify(xdsClient).watchClusterData(eq("bar.googleapis.com"), clusterWatcherCaptor2.capture()); - - ClusterWatcher clusterWatcher2 = clusterWatcherCaptor2.getValue(); - clusterWatcher2.onClusterChanged( - ClusterUpdate.newBuilder() - .setClusterName("bar.googleapis.com") - .setEdsServiceName("edsServiceBar.googleapis.com") - .setLbPolicy("round_robin") - .setLrsServerName("lrsBar.googleapis.com") - .build()); + @Deprecated + @Override + public NameResolver.Factory getNameResolverFactory() { + throw new UnsupportedOperationException("should not be called"); + } - assertThat(edsLbHelpers).hasSize(1); - assertThat(edsLoadBalancers).hasSize(1); - Helper edsLbHelper2 = edsLbHelpers.poll(); - LoadBalancer edsLoadBalancer2 = edsLoadBalancers.poll(); - ArgumentCaptor resolvedAddressesCaptor2 = ArgumentCaptor.forClass(null); - verify(edsLoadBalancer2).handleResolvedAddresses(resolvedAddressesCaptor2.capture()); - expectedEdsConfig = new EdsConfig( - "bar.googleapis.com", - "edsServiceBar.googleapis.com", - "lrsBar.googleapis.com", - roundRobinPolicy); - ResolvedAddresses resolvedAddressesBar = resolvedAddressesCaptor2.getValue(); - assertThat(resolvedAddressesBar.getLoadBalancingPolicyConfig()).isEqualTo(expectedEdsConfig); - assertThat(resolvedAddressesBar.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL)) - .isSameInstanceAs(xdsClientPool); - - SubchannelPicker picker2 = mock(SubchannelPicker.class); - edsLbHelper2.updateBalancingState(ConnectivityState.CONNECTING, picker2); - verify(helper, never()).updateBalancingState(ConnectivityState.CONNECTING, picker2); - verify(edsLoadBalancer1, never()).shutdown(); - - picker2 = mock(SubchannelPicker.class); - edsLbHelper2.updateBalancingState(ConnectivityState.READY, picker2); - verify(helper).updateBalancingState(ConnectivityState.READY, picker2); - verify(edsLoadBalancer1).shutdown(); - verify(xdsClient).cancelClusterDataWatch("foo.googleapis.com", clusterWatcher1); - - clusterWatcher2.onClusterChanged( - ClusterUpdate.newBuilder() - .setClusterName("bar.googleapis.com") - .setEdsServiceName("edsServiceBar2.googleapis.com") - .setLbPolicy("round_robin") - .build()); - verify(edsLoadBalancer2, times(2)).handleResolvedAddresses(resolvedAddressesCaptor2.capture()); - expectedEdsConfig = new EdsConfig( - "bar.googleapis.com", - "edsServiceBar2.googleapis.com", - null, - roundRobinPolicy); - ResolvedAddresses resolvedAddressesBar2 = resolvedAddressesCaptor2.getValue(); - assertThat(resolvedAddressesBar2.getLoadBalancingPolicyConfig()).isEqualTo(expectedEdsConfig); - - cdsLoadBalancer.shutdown(); - verify(edsLoadBalancer2).shutdown(); - verify(xdsClient).cancelClusterDataWatch("bar.googleapis.com", clusterWatcher2); - assertThat(xdsClientPool.xdsClient).isNull(); + @Override + public String getAuthority() { + return AUTHORITY; + } } - @Test - public void handleCdsConfigUpdate_withUpstreamTlsContext() { - assertThat(cdsLoadBalancer).isInstanceOf(CdsLoadBalancer.class); - ((CdsLoadBalancer)cdsLoadBalancer).setXdsSecurity(true); - assertThat(xdsClient).isNull(); - ResolvedAddresses resolvedAddresses1 = - ResolvedAddresses.newBuilder() - .setAddresses(ImmutableList.of()) - .setAttributes( - Attributes.newBuilder() - .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) - .build()) - .setLoadBalancingPolicyConfig(new CdsConfig("foo.googleapis.com")) - .build(); - cdsLoadBalancer.handleResolvedAddresses(resolvedAddresses1); - - ArgumentCaptor clusterWatcherCaptor1 = ArgumentCaptor.forClass(null); - verify(xdsClient).watchClusterData(eq("foo.googleapis.com"), clusterWatcherCaptor1.capture()); + private static final class FakeSubchannel extends Subchannel { + private final List eags; - UpstreamTlsContext upstreamTlsContext = - CommonTlsContextTestsUtil.buildUpstreamTlsContextFromFilenames( - CLIENT_KEY_FILE, CLIENT_PEM_FILE, CA_PEM_FILE); - - SslContextProvider mockSslContextProvider = mock(SslContextProvider.class); - doReturn(upstreamTlsContext).when(mockSslContextProvider).getUpstreamTlsContext(); - doReturn(mockSslContextProvider).when(mockTlsContextManager) - .findOrCreateClientSslContextProvider(same(upstreamTlsContext)); - - ClusterWatcher clusterWatcher1 = clusterWatcherCaptor1.getValue(); - clusterWatcher1.onClusterChanged( - ClusterUpdate.newBuilder() - .setClusterName("foo.googleapis.com") - .setEdsServiceName("edsServiceFoo.googleapis.com") - .setLbPolicy("round_robin") - .setUpstreamTlsContext(upstreamTlsContext) - .build()); + private FakeSubchannel(List eags) { + this.eags = eags; + } - assertThat(edsLbHelpers).hasSize(1); - assertThat(edsLoadBalancers).hasSize(1); - verify(mockTlsContextManager, never()) - .releaseClientSslContextProvider(any(SslContextProvider.class)); - Helper edsLbHelper1 = edsLbHelpers.poll(); - - ArrayList eagList = new ArrayList<>(); - eagList.add(new EquivalentAddressGroup(new InetSocketAddress("foo.com", 8080))); - eagList.add(new EquivalentAddressGroup(InetSocketAddress.createUnresolved("localhost", 8081), - Attributes.newBuilder().set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool).build())); - LoadBalancer.CreateSubchannelArgs createSubchannelArgs = - LoadBalancer.CreateSubchannelArgs.newBuilder() - .setAddresses(eagList) - .build(); - ArgumentCaptor createSubchannelArgsCaptor1 = - ArgumentCaptor.forClass(null); - verify(helper, never()) - .createSubchannel(any(LoadBalancer.CreateSubchannelArgs.class)); - edsLbHelper1.createSubchannel(createSubchannelArgs); - SslContextProviderSupplier sslContextProviderSupplier = - verifySslContextProviderSupplierAttribute(upstreamTlsContext, createSubchannelArgsCaptor1); - - // update with same upstreamTlsContext - reset(mockTlsContextManager); - clusterWatcher1.onClusterChanged( - ClusterUpdate.newBuilder() - .setClusterName("bar.googleapis.com") - .setEdsServiceName("eds1ServiceFoo.googleapis.com") - .setLbPolicy("round_robin") - .setUpstreamTlsContext(upstreamTlsContext) - .build()); + @Override + public void shutdown() { + } - verify(mockTlsContextManager, never()) - .releaseClientSslContextProvider(any(SslContextProvider.class)); - verify(mockTlsContextManager, never()).findOrCreateClientSslContextProvider( - any(UpstreamTlsContext.class)); - - // verify SslContextProviderSupplier.updateSslContext - doReturn(mockSslContextProvider) - .when(mockTlsContextManager) - .findOrCreateClientSslContextProvider(same(upstreamTlsContext)); - sslContextProviderSupplier.updateSslContext( - new SslContextProvider.Callback(MoreExecutors.directExecutor()) { - @Override - public void updateSecret(SslContext sslContext) {} - - @Override - protected void onException(Throwable throwable) {} - }); - verify(mockTlsContextManager, times(2)) - .findOrCreateClientSslContextProvider(eq(upstreamTlsContext)); - verify(mockTlsContextManager, never()) - .releaseClientSslContextProvider(eq(mockSslContextProvider)); - ArgumentCaptor callbackCaptor = ArgumentCaptor.forClass(null); - verify(mockSslContextProvider).addCallback(callbackCaptor.capture()); - SslContextProvider.Callback capturedCallback = callbackCaptor.getValue(); - assertThat(capturedCallback).isNotNull(); - capturedCallback.updateSecret(null); - verify(mockTlsContextManager).releaseClientSslContextProvider(eq(mockSslContextProvider)); - - // update with different upstreamTlsContext - reset(mockTlsContextManager); - reset(helper); - UpstreamTlsContext upstreamTlsContext1 = - CommonTlsContextTestsUtil.buildUpstreamTlsContextFromFilenames( - BAD_CLIENT_KEY_FILE, BAD_CLIENT_PEM_FILE, CA_PEM_FILE); - SslContextProvider mockSslContextProvider1 = mock(SslContextProvider.class); - doReturn(upstreamTlsContext1).when(mockSslContextProvider1).getUpstreamTlsContext(); - doReturn(mockSslContextProvider1).when(mockTlsContextManager) - .findOrCreateClientSslContextProvider(same(upstreamTlsContext1)); - clusterWatcher1.onClusterChanged( - ClusterUpdate.newBuilder() - .setClusterName("bar.googleapis.com") - .setEdsServiceName("eds1ServiceFoo.googleapis.com") - .setLbPolicy("round_robin") - .setUpstreamTlsContext(upstreamTlsContext1) - .build()); + @Override + public void requestConnection() { + } - verify(mockTlsContextManager).releaseClientSslContextProvider(same(mockSslContextProvider)); - verify(mockTlsContextManager, never()) - .findOrCreateClientSslContextProvider(same(upstreamTlsContext1)); - ArgumentCaptor createSubchannelArgsCaptor2 = - ArgumentCaptor.forClass(null); - edsLbHelper1.createSubchannel(createSubchannelArgs); - SslContextProviderSupplier sslContextProviderSupplier1 = - verifySslContextProviderSupplierAttribute(upstreamTlsContext1, createSubchannelArgsCaptor2); - // initialize sslContextProviderSupplier1 with sslContextProvider - sslContextProviderSupplier1.updateSslContext( - new SslContextProvider.Callback(MoreExecutors.directExecutor()) { - @Override - public void updateSecret(SslContext sslContext) {} - - @Override - protected void onException(Throwable throwable) {} - }); - - // update with null - reset(mockTlsContextManager); - reset(helper); - clusterWatcher1.onClusterChanged( - ClusterUpdate.newBuilder() - .setClusterName("bar.googleapis.com") - .setEdsServiceName("eds1ServiceFoo.googleapis.com") - .setLbPolicy("round_robin") - .setUpstreamTlsContext(null) - .build()); - verify(mockTlsContextManager).releaseClientSslContextProvider(same(mockSslContextProvider1)); - verify(mockTlsContextManager, never()).findOrCreateClientSslContextProvider( - any(UpstreamTlsContext.class)); - ArgumentCaptor createSubchannelArgsCaptor3 = - ArgumentCaptor.forClass(null); - edsLbHelper1.createSubchannel(createSubchannelArgs); - verifySslContextProviderSupplierAttribute(null, - createSubchannelArgsCaptor3); - - LoadBalancer edsLoadBalancer1 = edsLoadBalancers.poll(); - - cdsLoadBalancer.shutdown(); - verify(edsLoadBalancer1).shutdown(); - verify(xdsClient).cancelClusterDataWatch("foo.googleapis.com", clusterWatcher1); - assertThat(xdsClientPool.xdsClient).isNull(); - } + @Override + public List getAllAddresses() { + return eags; + } - private SslContextProviderSupplier verifySslContextProviderSupplierAttribute( - UpstreamTlsContext upstreamTlsContext, - ArgumentCaptor createSubchannelArgsCaptor1) { - verify(helper).createSubchannel(createSubchannelArgsCaptor1.capture()); - CreateSubchannelArgs capturedValue = createSubchannelArgsCaptor1.getValue(); - List capturedEagList = capturedValue.getAddresses(); - assertThat(capturedEagList.size()).isEqualTo(2); - EquivalentAddressGroup capturedEag = capturedEagList.get(0); - - SslContextProviderSupplier capturedSslContextProviderSupplier = - capturedEag.getAttributes().get(XdsAttributes.ATTR_SSL_CONTEXT_PROVIDER_SUPPLIER); - if (upstreamTlsContext == null) { - assertThat(capturedSslContextProviderSupplier).isNull(); - } else { - assertThat(capturedSslContextProviderSupplier).isNotNull(); - assertThat(capturedSslContextProviderSupplier.getUpstreamTlsContext()) - .isSameInstanceAs(upstreamTlsContext); - } - capturedEag = capturedEagList.get(1); - SslContextProviderSupplier capturedSslContextProviderSupplier1 = - capturedEag.getAttributes().get(XdsAttributes.ATTR_SSL_CONTEXT_PROVIDER_SUPPLIER); - assertThat(capturedSslContextProviderSupplier1) - .isSameInstanceAs(capturedSslContextProviderSupplier); - assertThat(capturedEag.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL)) - .isSameInstanceAs(xdsClientPool); - return capturedSslContextProviderSupplier; + @Override + public Attributes getAttributes() { + return Attributes.EMPTY; + } } - @Test - public void clusterWatcher_resourceNotExist() { - ResolvedAddresses resolvedAddresses = ResolvedAddresses.newBuilder() - .setAddresses(ImmutableList.of()) - .setAttributes(Attributes.newBuilder() - .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) - .build()) - .setLoadBalancingPolicyConfig(new CdsConfig("foo.googleapis.com")) - .build(); - cdsLoadBalancer.handleResolvedAddresses(resolvedAddresses); - - ArgumentCaptor clusterWatcherCaptor = ArgumentCaptor.forClass(null); - verify(xdsClient).watchClusterData(eq("foo.googleapis.com"), clusterWatcherCaptor.capture()); - - ClusterWatcher clusterWatcher = clusterWatcherCaptor.getValue(); - ArgumentCaptor pickerCaptor = ArgumentCaptor.forClass(null); - clusterWatcher.onResourceDoesNotExist("foo.googleapis.com"); - assertThat(edsLoadBalancers).isEmpty(); - verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); - PickResult result = pickerCaptor.getValue().pickSubchannel(mock(PickSubchannelArgs.class)); - assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAVAILABLE); - assertThat(result.getStatus().getDescription()) - .isEqualTo("Resource foo.googleapis.com is unavailable"); - } + private static final class FakeTlsContextManager implements TlsContextManager { - @Test - public void clusterWatcher_resourceRemoved() { - ResolvedAddresses resolvedAddresses = ResolvedAddresses.newBuilder() - .setAddresses(ImmutableList.of()) - .setAttributes(Attributes.newBuilder() - .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) - .build()) - .setLoadBalancingPolicyConfig(new CdsConfig("foo.googleapis.com")) - .build(); - cdsLoadBalancer.handleResolvedAddresses(resolvedAddresses); - - ArgumentCaptor clusterWatcherCaptor = ArgumentCaptor.forClass(null); - verify(xdsClient).watchClusterData(eq("foo.googleapis.com"), clusterWatcherCaptor.capture()); - - ClusterWatcher clusterWatcher = clusterWatcherCaptor.getValue(); - ArgumentCaptor pickerCaptor = ArgumentCaptor.forClass(null); - clusterWatcher.onClusterChanged( - ClusterUpdate.newBuilder() - .setClusterName("foo.googleapis.com") - .setEdsServiceName("edsServiceFoo.googleapis.com") - .setLbPolicy("round_robin") - .build()); - assertThat(edsLoadBalancers).hasSize(1); - assertThat(edsLbHelpers).hasSize(1); - LoadBalancer edsLoadBalancer = edsLoadBalancers.poll(); - Helper edsHelper = edsLbHelpers.poll(); - SubchannelPicker subchannelPicker = mock(SubchannelPicker.class); - edsHelper.updateBalancingState(READY, subchannelPicker); - verify(helper).updateBalancingState(eq(READY), same(subchannelPicker)); - - clusterWatcher.onResourceDoesNotExist("foo.googleapis.com"); - verify(edsLoadBalancer).shutdown(); - verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); - PickResult result = pickerCaptor.getValue().pickSubchannel(mock(PickSubchannelArgs.class)); - assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAVAILABLE); - assertThat(result.getStatus().getDescription()) - .isEqualTo("Resource foo.googleapis.com is unavailable"); - } + @Override + public SslContextProvider findOrCreateClientSslContextProvider( + UpstreamTlsContext upstreamTlsContext) { + SslContextProvider sslContextProvider = mock(SslContextProvider.class); + when(sslContextProvider.getUpstreamTlsContext()).thenReturn(upstreamTlsContext); + return sslContextProvider; + } - @Test - public void clusterWatcher_onErrorCalledBeforeAndAfterOnClusterChanged() { - ResolvedAddresses resolvedAddresses = ResolvedAddresses.newBuilder() - .setAddresses(ImmutableList.of()) - .setAttributes(Attributes.newBuilder() - .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) - .build()) - .setLoadBalancingPolicyConfig(new CdsConfig("foo.googleapis.com")) - .build(); - cdsLoadBalancer.handleResolvedAddresses(resolvedAddresses); - - ArgumentCaptor clusterWatcherCaptor = ArgumentCaptor.forClass(null); - verify(xdsClient).watchClusterData(eq("foo.googleapis.com"), clusterWatcherCaptor.capture()); - - ClusterWatcher clusterWatcher = clusterWatcherCaptor.getValue(); - - // Call onError() before onClusterChanged() ever called. - clusterWatcher.onError(Status.DATA_LOSS.withDescription("fake status")); - assertThat(edsLoadBalancers).isEmpty(); - verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); - - clusterWatcher.onClusterChanged( - ClusterUpdate.newBuilder() - .setClusterName("foo.googleapis.com") - .setEdsServiceName("edsServiceFoo.googleapis.com") - .setLbPolicy("round_robin") - .build()); + @Override + public SslContextProvider releaseClientSslContextProvider( + SslContextProvider sslContextProvider) { + // no-op + return null; + } - assertThat(edsLbHelpers).hasSize(1); - assertThat(edsLoadBalancers).hasSize(1); - Helper edsLbHelper = edsLbHelpers.poll(); - LoadBalancer edsLoadBalancer = edsLoadBalancers.poll(); - verify(edsLoadBalancer).handleResolvedAddresses(any(ResolvedAddresses.class)); - SubchannelPicker picker = mock(SubchannelPicker.class); - - edsLbHelper.updateBalancingState(ConnectivityState.READY, picker); - verify(helper).updateBalancingState(ConnectivityState.READY, picker); - - // Call onError() after onClusterChanged(). - clusterWatcher.onError(Status.DATA_LOSS.withDescription("fake status")); - // Verify no more TRANSIENT_FAILURE. - verify(helper, times(1)) - .updateBalancingState(eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); + @Override + public SslContextProvider findOrCreateServerSslContextProvider( + DownstreamTlsContext downstreamTlsContext) { + throw new UnsupportedOperationException("should not be called"); + } + + @Override + public SslContextProvider releaseServerSslContextProvider( + SslContextProvider sslContextProvider) { + throw new UnsupportedOperationException("should not be called"); + } } } From c9195949625aaacfea9fcb7a244427670c23bb89 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Wed, 9 Sep 2020 10:07:48 -0700 Subject: [PATCH 27/86] netty-shaded: Fix publish regression for javadoc and sources 96ad6338 accidentally caused the javadoc and sources jars to no longer be published for grpc-netty-shaded. It would appear to be due to the jars being empty. This commit causes them to be published again. --- netty/shaded/build.gradle | 3 +++ 1 file changed, 3 insertions(+) diff --git a/netty/shaded/build.gradle b/netty/shaded/build.gradle index 904bccd44df..5226fb60ab2 100644 --- a/netty/shaded/build.gradle +++ b/netty/shaded/build.gradle @@ -47,6 +47,9 @@ publishing { maven(MavenPublication) { // Ideally swap to project.shadow.component(it) when it isn't broken for project deps artifact shadowJar + // Empty jars are not published via withJavadocJar() and withSourcesJar() + artifact javadocJar + artifact sourcesJar pom.withXml { def dependencies = asNode().appendNode('dependencies') From 7c7c4a7daadf3c0605a6d162b161a3729dda9eb1 Mon Sep 17 00:00:00 2001 From: sanjaypujare Date: Wed, 9 Sep 2020 16:22:34 -0700 Subject: [PATCH 28/86] xds: accept all forms of gkeClusterUrl and fix the 'x-goog-request-params' value (#7403) --- .../MeshCaCertificateProvider.java | 2 +- .../MeshCaCertificateProviderProvider.java | 2 +- .../CommonCertProviderTestUtils.java | 28 ++++++++++++ ...MeshCaCertificateProviderProviderTest.java | 44 ++++++++++++++++++- .../MeshCaCertificateProviderTest.java | 2 +- 5 files changed, 73 insertions(+), 5 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProvider.java b/xds/src/main/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProvider.java index c6fa9a39b60..efb9a54b82a 100644 --- a/xds/src/main/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProvider.java +++ b/xds/src/main/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProvider.java @@ -457,7 +457,7 @@ public ClientCall interceptCall( @Override public void start(Listener responseListener, Metadata headers) { - headers.put(KEY_FOR_ZONE_INFO, zone); + headers.put(KEY_FOR_ZONE_INFO, "location=locations/" + zone); super.start(responseListener, headers); } }; diff --git a/xds/src/main/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderProvider.java b/xds/src/main/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderProvider.java index c29170b104d..fc04e90ec27 100644 --- a/xds/src/main/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderProvider.java +++ b/xds/src/main/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderProvider.java @@ -69,7 +69,7 @@ final class MeshCaCertificateProviderProvider implements CertificateProviderProv static final long RPC_TIMEOUT_SECONDS = 10L; private static final Pattern CLUSTER_URL_PATTERN = Pattern - .compile(".*/projects/(.*)/locations/(.*)/clusters/.*"); + .compile(".*/projects/(.*)/(?:locations|zones)/(.*)/clusters/.*"); private static final String TRUST_DOMAIN_SUFFIX = ".svc.id.goog"; private static final String AUDIENCE_PREFIX = "identitynamespace:"; diff --git a/xds/src/test/java/io/grpc/xds/internal/certprovider/CommonCertProviderTestUtils.java b/xds/src/test/java/io/grpc/xds/internal/certprovider/CommonCertProviderTestUtils.java index ce494e5efa9..4ca3bc1b138 100644 --- a/xds/src/test/java/io/grpc/xds/internal/certprovider/CommonCertProviderTestUtils.java +++ b/xds/src/test/java/io/grpc/xds/internal/certprovider/CommonCertProviderTestUtils.java @@ -163,6 +163,34 @@ static Bootstrapper.BootstrapInfo getMinimalBootstrapInfo() throws IOException { return Bootstrapper.parseConfig(rawData); } + static Bootstrapper.BootstrapInfo getMinimalBootstrapInfo_v1beta1AndZone() throws IOException { + String rawData = + "{\n" + + " \"xds_servers\": [],\n" + + " \"certificate_providers\": {\n" + + " \"gcp_id\": {\n" + + " \"plugin_name\": \"testca\",\n" + + " \"config\": {\n" + + " \"server\": {\n" + + " \"api_type\": \"GRPC\",\n" + + " \"grpc_services\": [{\n" + + " \"google_grpc\": {\n" + + " \"call_credentials\": [{\n" + + " \"sts_service\": {\n" + + " \"subject_token_path\": \"/tmp/path5\"\n" + + " }\n" + + " }]\n" // end call_credentials + + " }\n" // end google_grpc + + " }]\n" // end grpc_services + + " },\n" // end server + + " \"location\": \"https://container.googleapis.com/v1beta1/projects/test-project1/zones/test-zone2/clusters/test-cluster3\"\n" + + " }\n" // end config + + " }\n" // end gcp_id + + " }\n" + + "}"; + return Bootstrapper.parseConfig(rawData); + } + static Bootstrapper.BootstrapInfo getMinimalAndBadClusterUrlBootstrapInfo() throws IOException { String rawData = "{\n" diff --git a/xds/src/test/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderProviderTest.java b/xds/src/test/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderProviderTest.java index 123db14a059..0f92d19cff8 100644 --- a/xds/src/test/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderProviderTest.java @@ -49,10 +49,10 @@ public class MeshCaCertificateProviderProviderTest { public static final String EXPECTED_AUDIENCE = "identitynamespace:test-project1.svc.id.goog:https://container.googleapis.com/v1/projects/test-project1/locations/test-zone2/clusters/test-cluster3"; + public static final String EXPECTED_AUDIENCE_V1BETA1_ZONE = + "identitynamespace:test-project1.svc.id.goog:https://container.googleapis.com/v1beta1/projects/test-project1/zones/test-zone2/clusters/test-cluster3"; public static final String TMP_PATH_4 = "/tmp/path4"; public static final String NON_DEFAULT_MESH_CA_URL = "nonDefaultMeshCaUrl"; - public static final String GKE_CLUSTER_URL = - "https://container.googleapis.com/v1/projects/test-project1/locations/test-zone2/clusters/test-cluster3"; @Mock StsCredentials.Factory stsCredentialsFactory; @@ -140,6 +140,41 @@ public void createProvider_minimalConfig() throws IOException { eq(TimeUnit.SECONDS.toMillis(RPC_TIMEOUT_SECONDS))); } + @Test + public void createProvider_minimalConfig_v1beta1AndZone() throws IOException { + CertificateProvider.DistributorWatcher distWatcher = + new CertificateProvider.DistributorWatcher(); + Map map = buildMinimalConfig_v1beta1AndZone(); + ScheduledExecutorService mockService = mock(ScheduledExecutorService.class); + when(scheduledExecutorServiceFactory.create( + eq(MeshCaCertificateProviderProvider.MESHCA_URL_DEFAULT))) + .thenReturn(mockService); + provider.createCertificateProvider(map, distWatcher, true); + verify(stsCredentialsFactory, times(1)) + .create( + eq(MeshCaCertificateProviderProvider.STS_URL_DEFAULT), + eq(EXPECTED_AUDIENCE_V1BETA1_ZONE), + eq("/tmp/path5")); + verify(meshCaCertificateProviderFactory, times(1)) + .create( + eq(distWatcher), + eq(true), + eq(MeshCaCertificateProviderProvider.MESHCA_URL_DEFAULT), + eq("test-zone2"), + eq(MeshCaCertificateProviderProvider.CERT_VALIDITY_SECONDS_DEFAULT), + eq(MeshCaCertificateProviderProvider.KEY_SIZE_DEFAULT), + eq(MeshCaCertificateProviderProvider.KEY_ALGO_DEFAULT), + eq(MeshCaCertificateProviderProvider.SIGNATURE_ALGO_DEFAULT), + eq(meshCaChannelFactory), + eq(backoffPolicyProvider), + eq(MeshCaCertificateProviderProvider.RENEWAL_GRACE_PERIOD_SECONDS_DEFAULT), + eq(MeshCaCertificateProviderProvider.MAX_RETRY_ATTEMPTS_DEFAULT), + (GoogleCredentials) isNull(), + eq(mockService), + eq(timeProvider), + eq(TimeUnit.SECONDS.toMillis(RPC_TIMEOUT_SECONDS))); + } + @Test public void createProvider_missingGkeUrl_expectException() throws IOException { CertificateProvider.DistributorWatcher distWatcher = @@ -234,6 +269,11 @@ public void createProvider_nonDefaultFullConfig() throws IOException { return getCertProviderConfig(CommonCertProviderTestUtils.getMinimalBootstrapInfo()); } + private static Map buildMinimalConfig_v1beta1AndZone() throws IOException { + return getCertProviderConfig( + CommonCertProviderTestUtils.getMinimalBootstrapInfo_v1beta1AndZone()); + } + private static Map buildBadClusterUrlConfig() throws IOException { return getCertProviderConfig( CommonCertProviderTestUtils.getMinimalAndBadClusterUrlBootstrapInfo()); diff --git a/xds/src/test/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderTest.java b/xds/src/test/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderTest.java index 3ead9134007..5698b66dbe9 100644 --- a/xds/src/test/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderTest.java @@ -534,7 +534,7 @@ private void verifyReceivedMetadataValues(int count) { assertThat(receivedZoneValues).hasSize(count); for (int i = 0; i < count; i++) { assertThat(receivedStsCreds.poll()).isEqualTo("Bearer " + TEST_STS_TOKEN + i); - assertThat(receivedZoneValues.poll()).isEqualTo("us-west2-a"); + assertThat(receivedZoneValues.poll()).isEqualTo("location=locations/us-west2-a"); } } From 1411e6f61e6e2d3e7f846b8d89bc8926617bf0dc Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Wed, 9 Sep 2020 17:47:10 -0700 Subject: [PATCH 29/86] core: fix pending call not drained when shutdown There was bug that new pending calls were not drained after channel is shutdown. The bug was worked around by #7354 . Fixing by making sure new calls fail immediately if the channel is already shutdown. --- .../io/grpc/internal/ManagedChannelImpl.java | 100 +++++++++++++----- .../grpc/internal/ManagedChannelImplTest.java | 35 ++++++ 2 files changed, 108 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java b/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java index 9af74d046bf..2a46c1e31d2 100644 --- a/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java +++ b/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java @@ -260,7 +260,7 @@ public void uncaughtException(Thread t, Throwable e) { private final ChannelTracer channelTracer; private final ChannelLogger channelLogger; private final InternalChannelz channelz; - + private final RealChannel realChannel; // Must be mutated and read from syncContext // a flag for doing channel tracing when flipped private ResolutionState lastResolutionState = ResolutionState.NO_RESOLUTION; @@ -655,8 +655,8 @@ public void execute(Runnable command) { this.defaultServiceConfig = null; } this.lookUpServiceConfig = builder.lookUpServiceConfig; - Channel channel = new RealChannel(nameResolver.getServiceAuthority()); - channel = ClientInterceptors.intercept(channel, serviceConfigInterceptor); + realChannel = new RealChannel(nameResolver.getServiceAuthority()); + Channel channel = ClientInterceptors.intercept(realChannel, serviceConfigInterceptor); if (builder.binlog != null) { channel = builder.binlog.wrapChannel(channel); } @@ -773,11 +773,6 @@ public ManagedChannelImpl shutdown() { if (!shutdown.compareAndSet(false, true)) { return this; } - - // Put gotoState(SHUTDOWN) as early into the syncContext's queue as possible. - // delayedTransport.shutdown() may also add some tasks into the queue. But some things inside - // delayedTransport.shutdown() like setting delayedTransport.shutdown = true are not run in the - // syncContext's queue and should not be blocked, so we do not drain() immediately here. final class Shutdown implements Runnable { @Override public void run() { @@ -786,9 +781,8 @@ public void run() { } } - syncContext.executeLater(new Shutdown()); - - uncommittedRetriableStreamsRegistry.onShutdown(SHUTDOWN_STATUS); + syncContext.execute(new Shutdown()); + realChannel.shutdown(); final class CancelIdleTimer implements Runnable { @Override public void run() { @@ -809,7 +803,7 @@ public void run() { public ManagedChannelImpl shutdownNow() { channelLogger.log(ChannelLogLevel.DEBUG, "shutdownNow() called"); shutdown(); - uncommittedRetriableStreamsRegistry.onShutdownNow(SHUTDOWN_NOW_STATUS); + realChannel.shutdownNow(); final class ShutdownNow implements Runnable { @Override public void run() { @@ -918,9 +912,6 @@ private RealChannel(String authority) { @Override public ClientCall newCall( MethodDescriptor method, CallOptions callOptions) { - if (true) { // FIXME(zdapeng): there is a bug for using PendingCall. Temporarily disable it. - return newClientCall(method, callOptions); - } if (configSelector.get() != INITIAL_PENDING_SELECTOR) { return newClientCall(method, callOptions); } @@ -936,6 +927,23 @@ public void run() { // tests might observe slight behavior difference from earlier grpc versions. return newClientCall(method, callOptions); } + if (shutdown.get()) { + // Return a failing ClientCall. + return new ClientCall() { + @Override + public void start(Listener responseListener, Metadata headers) { + responseListener.onClose(SHUTDOWN_STATUS, new Metadata()); + } + + @Override public void request(int numMessages) {} + + @Override public void cancel(@Nullable String message, @Nullable Throwable cause) {} + + @Override public void halfClose() {} + + @Override public void sendMessage(ReqT message) {} + }; + } Context context = Context.current(); final PendingCall pendingCall = new PendingCall<>(context, method, callOptions); syncContext.execute(new Runnable() { @@ -955,6 +963,51 @@ public void run() { return pendingCall; } + // Must run in SynchronizationContext. + private void drainPendingCalls() { + if (pendingCalls == null) { + return; + } + for (RealChannel.PendingCall pendingCall : pendingCalls) { + pendingCall.reprocess(); + } + } + + void shutdown() { + final class RealChannelShutdown implements Runnable { + @Override + public void run() { + if (pendingCalls == null) { + if (configSelector.get() == INITIAL_PENDING_SELECTOR) { + configSelector.set(null); + } + uncommittedRetriableStreamsRegistry.onShutdown(SHUTDOWN_STATUS); + } + } + } + + syncContext.execute(new RealChannelShutdown()); + } + + void shutdownNow() { + final class RealChannelShutdownNow implements Runnable { + @Override + public void run() { + if (configSelector.get() == INITIAL_PENDING_SELECTOR) { + configSelector.set(null); + } + if (pendingCalls != null) { + for (RealChannel.PendingCall pendingCall : pendingCalls) { + pendingCall.cancel("Channel is forcefully shutdown", null); + } + } + uncommittedRetriableStreamsRegistry.onShutdownNow(SHUTDOWN_NOW_STATUS); + } + } + + syncContext.execute(new RealChannelShutdownNow()); + } + @Override public String authority() { return authority; @@ -1007,6 +1060,9 @@ public void run() { if (pendingCalls.isEmpty()) { inUseStateAggregator.updateObjectInUse(pendingCallsInUseObject, false); pendingCalls = null; + if (shutdown.get()) { + uncommittedRetriableStreamsRegistry.onShutdown(SHUTDOWN_STATUS); + } } } } @@ -1583,7 +1639,7 @@ public void run() { re); } } - drainPendingCalls(); + realChannel.drainPendingCalls(); Attributes effectiveAttrs = resolutionResult.getAttributes(); // Call LB only if it's not shutdown. If LB is shutdown, lbHelper won't match. @@ -1633,7 +1689,7 @@ private void handleErrorInSyncContext(Status error) { new Object[] {getLogId(), error}); if (configSelector.get() == INITIAL_PENDING_SELECTOR) { configSelector.set(null); - drainPendingCalls(); + realChannel.drainPendingCalls(); } if (lastResolutionState != ResolutionState.ERROR) { channelLogger.log(ChannelLogLevel.WARNING, "Failed to resolve name: {0}", error); @@ -1649,16 +1705,6 @@ private void handleErrorInSyncContext(Status error) { scheduleExponentialBackOffInSyncContext(); } - // Must run in SynchronizationContext. - private void drainPendingCalls() { - if (pendingCalls == null) { - return; - } - for (RealChannel.PendingCall pendingCall : pendingCalls) { - pendingCall.reprocess(); - } - } - private void scheduleExponentialBackOffInSyncContext() { if (scheduledNameResolverRefresh != null && scheduledNameResolverRefresh.isPending()) { // The name resolver may invoke onError multiple times, but we only want to diff --git a/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java b/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java index ba17deba613..fdebd545ff0 100644 --- a/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java +++ b/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java @@ -551,6 +551,41 @@ public void shutdownWithNoTransportsEverCreated() { verify(executorPool).returnObject(executor.getScheduledExecutorService()); } + @Test + public void shutdownNow_pendingCallShouldFail() { + channelBuilder.nameResolverFactory( + new FakeNameResolverFactory.Builder(expectedUri) + .setResolvedAtStart(false) + .setServers(Collections.singletonList(new EquivalentAddressGroup(socketAddress))) + .build()); + createChannel(); + ClientCall call = channel.newCall(method, CallOptions.DEFAULT); + call.start(mockCallListener, new Metadata()); + channel.shutdown(); + executor.runDueTasks(); + verify(mockCallListener, never()).onClose(any(Status.class), any(Metadata.class)); + channel.shutdownNow(); + executor.runDueTasks(); + verify(mockCallListener).onClose(statusCaptor.capture(), any(Metadata.class)); + assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.CANCELLED); + } + + @Test + public void shutdownWithNoNameResolution_newCallShouldFail() { + channelBuilder.nameResolverFactory( + new FakeNameResolverFactory.Builder(expectedUri) + .setResolvedAtStart(false) + .setServers(Collections.singletonList(new EquivalentAddressGroup(socketAddress))) + .build()); + createChannel(); + channel.shutdown(); + ClientCall call = channel.newCall(method, CallOptions.DEFAULT); + call.start(mockCallListener, new Metadata()); + executor.runDueTasks(); + verify(mockCallListener).onClose(statusCaptor.capture(), any(Metadata.class)); + assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); + } + @Test public void channelzMembership() throws Exception { createChannel(); From 2f60c0a66c339ba83ce981b3931e531ac5eed654 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 20 Aug 2020 09:22:20 -0700 Subject: [PATCH 30/86] Revert "Call Cipher APIs with non-direct ByteBuffers and perform copies in the ALTS code. (cl/308901367)" This reverts commit a7bca2305363dd9dbc8d7f5d1c97dca464db53ca. --- .../alts/internal/AltsChannelCrypter.java | 99 +++++++++---------- 1 file changed, 44 insertions(+), 55 deletions(-) diff --git a/alts/src/main/java/io/grpc/alts/internal/AltsChannelCrypter.java b/alts/src/main/java/io/grpc/alts/internal/AltsChannelCrypter.java index e47433ff034..5e4c6fec301 100644 --- a/alts/src/main/java/io/grpc/alts/internal/AltsChannelCrypter.java +++ b/alts/src/main/java/io/grpc/alts/internal/AltsChannelCrypter.java @@ -17,10 +17,10 @@ package io.grpc.alts.internal; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Verify.verify; import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; import java.nio.ByteBuffer; import java.security.GeneralSecurityException; import java.util.List; @@ -56,72 +56,61 @@ static int getCounterLength() { @Override public void encrypt(ByteBuf outBuf, List plainBufs) throws GeneralSecurityException { - byte[] tempArr = new byte[outBuf.writableBytes()]; - - // Copy plaintext into tempArr. - { - ByteBuf tempBuf = Unpooled.wrappedBuffer(tempArr, 0, tempArr.length - TAG_LENGTH); - tempBuf.resetWriterIndex(); - for (ByteBuf plainBuf : plainBufs) { - tempBuf.writeBytes(plainBuf); - } + checkArgument(outBuf.nioBufferCount() == 1); + // Copy plaintext buffers into outBuf for in-place encryption on single direct buffer. + ByteBuf plainBuf = outBuf.slice(outBuf.writerIndex(), outBuf.writableBytes()); + plainBuf.writerIndex(0); + for (ByteBuf inBuf : plainBufs) { + plainBuf.writeBytes(inBuf); } - // Encrypt into tempArr. - { - ByteBuffer out = ByteBuffer.wrap(tempArr); - ByteBuffer plain = ByteBuffer.wrap(tempArr, 0, tempArr.length - TAG_LENGTH); - - byte[] counter = incrementOutCounter(); - aeadCrypter.encrypt(out, plain, counter); - } - outBuf.writeBytes(tempArr); + verify(outBuf.writableBytes() == plainBuf.readableBytes() + TAG_LENGTH); + ByteBuffer out = outBuf.internalNioBuffer(outBuf.writerIndex(), outBuf.writableBytes()); + ByteBuffer plain = out.duplicate(); + plain.limit(out.limit() - TAG_LENGTH); + + byte[] counter = incrementOutCounter(); + int outPosition = out.position(); + aeadCrypter.encrypt(out, plain, counter); + int bytesWritten = out.position() - outPosition; + outBuf.writerIndex(outBuf.writerIndex() + bytesWritten); + verify(!outBuf.isWritable()); } @Override - public void decrypt(ByteBuf outBuf, ByteBuf tagBuf, List ciphertextBufs) + public void decrypt(ByteBuf out, ByteBuf tag, List ciphertextBufs) throws GeneralSecurityException { - // There is enough space for the ciphertext including the tag in outBuf. - byte[] tempArr = new byte[outBuf.writableBytes()]; - - // Copy ciphertext and tag into tempArr. - { - ByteBuf tempBuf = Unpooled.wrappedBuffer(tempArr); - tempBuf.resetWriterIndex(); - for (ByteBuf ciphertextBuf : ciphertextBufs) { - tempBuf.writeBytes(ciphertextBuf); - } - tempBuf.writeBytes(tagBuf); - } - decryptInternal(outBuf, tempArr); - } + ByteBuf cipherTextAndTag = out.slice(out.writerIndex(), out.writableBytes()); + cipherTextAndTag.writerIndex(0); - @Override - public void decrypt( - ByteBuf outBuf, ByteBuf ciphertextAndTagDirect) throws GeneralSecurityException { - byte[] tempArr = new byte[ciphertextAndTagDirect.readableBytes()]; - - // Copy ciphertext and tag into tempArr. - { - ByteBuf tempBuf = Unpooled.wrappedBuffer(tempArr); - tempBuf.resetWriterIndex(); - tempBuf.writeBytes(ciphertextAndTagDirect); + for (ByteBuf inBuf : ciphertextBufs) { + cipherTextAndTag.writeBytes(inBuf); } + cipherTextAndTag.writeBytes(tag); - decryptInternal(outBuf, tempArr); + decrypt(out, cipherTextAndTag); } - private void decryptInternal(ByteBuf outBuf, byte[] tempArr) throws GeneralSecurityException { - // Perform in-place decryption on tempArr. - { - ByteBuffer ciphertextAndTag = ByteBuffer.wrap(tempArr); - ByteBuffer out = ByteBuffer.wrap(tempArr); - byte[] counter = incrementInCounter(); - aeadCrypter.decrypt(out, ciphertextAndTag, counter); - } - - outBuf.writeBytes(tempArr, 0, tempArr.length - TAG_LENGTH); + @Override + public void decrypt(ByteBuf out, ByteBuf ciphertextAndTag) throws GeneralSecurityException { + int bytesRead = ciphertextAndTag.readableBytes(); + checkArgument(bytesRead == out.writableBytes()); + + checkArgument(out.nioBufferCount() == 1); + ByteBuffer outBuffer = out.internalNioBuffer(out.writerIndex(), out.writableBytes()); + + checkArgument(ciphertextAndTag.nioBufferCount() == 1); + ByteBuffer ciphertextAndTagBuffer = + ciphertextAndTag.nioBuffer(ciphertextAndTag.readerIndex(), bytesRead); + + byte[] counter = incrementInCounter(); + int outPosition = outBuffer.position(); + aeadCrypter.decrypt(outBuffer, ciphertextAndTagBuffer, counter); + int bytesWritten = outBuffer.position() - outPosition; + out.writerIndex(out.writerIndex() + bytesWritten); + ciphertextAndTag.readerIndex(out.readerIndex() + bytesRead); + verify(out.writableBytes() == TAG_LENGTH); } @Override From dca74af870495c7aaa92c882e8d5b5a162762732 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 20 Aug 2020 09:24:58 -0700 Subject: [PATCH 31/86] Upgrade Conscrypt to 2.5.1 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index df5826c8628..67750629a2f 100644 --- a/build.gradle +++ b/build.gradle @@ -174,7 +174,7 @@ subprojects { // examples/example-tls/pom.xml netty_tcnative: 'io.netty:netty-tcnative-boringssl-static:2.0.31.Final', - conscrypt: 'org.conscrypt:conscrypt-openjdk-uber:2.2.1', + conscrypt: 'org.conscrypt:conscrypt-openjdk-uber:2.5.1', re2j: 'com.google.re2j:re2j:1.2', bouncycastle: 'org.bouncycastle:bcpkix-jdk15on:1.61', From f3a1a3ff10ee07383aad3a51afb3fbbc7fe3d0db Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Thu, 10 Sep 2020 10:32:46 -0700 Subject: [PATCH 32/86] core: refactor handle config update with ConfigSelector Get rid of `ServiceConfigInterceptor` and handle config update with `ConfigSelector` inside `ClientCallImpl`. --- .../java/io/grpc/internal/ClientCallImpl.java | 98 +++- .../java/io/grpc/internal/HedgingPolicy.java | 15 - .../io/grpc/internal/ManagedChannelImpl.java | 44 +- .../internal/ManagedChannelServiceConfig.java | 49 +- .../io/grpc/internal/RetriableStream.java | 37 +- .../java/io/grpc/internal/RetryPolicy.java | 15 - .../internal/ServiceConfigInterceptor.java | 206 ------- .../io/grpc/internal/ClientCallImplTest.java | 163 +++++- .../io/grpc/internal/HedgingPolicyTest.java | 76 +-- .../grpc/internal/ManagedChannelImplTest.java | 84 ++- .../ManagedChannelServiceConfigTest.java | 84 +++ .../io/grpc/internal/RetriableStreamTest.java | 18 +- .../io/grpc/internal/RetryPolicyTest.java | 75 +-- .../ServiceConfigInterceptorTest.java | 528 ------------------ 14 files changed, 539 insertions(+), 953 deletions(-) delete mode 100644 core/src/main/java/io/grpc/internal/ServiceConfigInterceptor.java delete mode 100644 core/src/test/java/io/grpc/internal/ServiceConfigInterceptorTest.java diff --git a/core/src/main/java/io/grpc/internal/ClientCallImpl.java b/core/src/main/java/io/grpc/internal/ClientCallImpl.java index 45029e9e649..db01fad6b3a 100644 --- a/core/src/main/java/io/grpc/internal/ClientCallImpl.java +++ b/core/src/main/java/io/grpc/internal/ClientCallImpl.java @@ -40,12 +40,15 @@ import io.grpc.Context.CancellationListener; import io.grpc.Deadline; import io.grpc.DecompressorRegistry; +import io.grpc.ForwardingClientCallListener.SimpleForwardingClientCallListener; import io.grpc.InternalConfigSelector; import io.grpc.InternalDecompressorRegistry; +import io.grpc.LoadBalancer.PickSubchannelArgs; import io.grpc.Metadata; import io.grpc.MethodDescriptor; import io.grpc.MethodDescriptor.MethodType; import io.grpc.Status; +import io.grpc.internal.ManagedChannelServiceConfig.MethodInfo; import io.perfmark.Link; import io.perfmark.PerfMark; import io.perfmark.Tag; @@ -83,7 +86,7 @@ final class ClientCallImpl extends ClientCall { private final CallTracer channelCallsTracer; private final Context context; private final boolean unaryRequest; - private final CallOptions callOptions; + private CallOptions callOptions; private ClientStream stream; private volatile boolean cancelListenersShouldBeRemoved; private boolean cancelCalled; @@ -91,6 +94,8 @@ final class ClientCallImpl extends ClientCall { private final ClientStreamProvider clientStreamProvider; private ContextCancellationListener cancellationListener; private final ScheduledExecutorService deadlineCancellationExecutor; + @Nullable + private final InternalConfigSelector configSelector; private boolean fullStreamDecompression; private DecompressorRegistry decompressorRegistry = DecompressorRegistry.getDefaultInstance(); private CompressorRegistry compressorRegistry = CompressorRegistry.getDefaultInstance(); @@ -103,7 +108,7 @@ final class ClientCallImpl extends ClientCall { ClientStreamProvider clientStreamProvider, ScheduledExecutorService deadlineCancellationExecutor, CallTracer channelCallsTracer, - InternalConfigSelector configSelector) { + @Nullable InternalConfigSelector configSelector) { this.method = method; // TODO(carl-mastrangelo): consider moving this construction to ManagedChannelImpl. this.tag = PerfMark.createTag(method.getFullMethodName(), System.identityHashCode(this)); @@ -125,6 +130,7 @@ final class ClientCallImpl extends ClientCall { this.callOptions = callOptions; this.clientStreamProvider = clientStreamProvider; this.deadlineCancellationExecutor = deadlineCancellationExecutor; + this.configSelector = configSelector; PerfMark.event("ClientCall.", tag); } @@ -207,7 +213,7 @@ public void start(Listener observer, Metadata headers) { } } - private void startInternal(final Listener observer, Metadata headers) { + private void startInternal(Listener observer, Metadata headers) { checkState(stream == null, "Already started"); checkState(!cancelCalled, "call was cancelled"); checkNotNull(observer, "observer"); @@ -220,7 +226,25 @@ private void startInternal(final Listener observer, Metadata headers) { executeCloseObserverInContext(observer, statusFromCancelled(context)); return; } - // TODO(zdapeng): configSelector.selectConfig() + + if (configSelector != null) { + PickSubchannelArgs args = new PickSubchannelArgsImpl(method, headers, callOptions); + InternalConfigSelector.Result result = configSelector.selectConfig(args); + Status status = result.getStatus(); + if (!status.isOk()) { + executeCloseObserverInContext(observer, status); + return; + } + callOptions = result.getCallOptions(); + Runnable committedCallback = result.getCommittedCallback(); + if (committedCallback != null) { + observer = new CommittedCallbackListener(observer, committedCallback); + } + ManagedChannelServiceConfig config = (ManagedChannelServiceConfig) result.getConfig(); + MethodInfo methodInfo = config.getMethodConfig(method); + applyMethodConfig(methodInfo); + } + final String compressorName = callOptions.getCompressor(); Compressor compressor; if (compressorName != null) { @@ -297,6 +321,72 @@ private void startInternal(final Listener observer, Metadata headers) { } } + private final class CommittedCallbackListener extends + SimpleForwardingClientCallListener { + final Runnable committedCallback; + boolean committed; + + CommittedCallbackListener(Listener delegate, Runnable committedCallback) { + super(delegate); + this.committedCallback = committedCallback; + } + + @Override + public void onHeaders(Metadata headers) { + committed = true; + committedCallback.run(); + delegate().onHeaders(headers); + } + + @Override + public void onClose(Status status, Metadata trailers) { + if (!committed) { + committed = true; + committedCallback.run(); + } + delegate().onClose(status, trailers); + } + } + + private void applyMethodConfig(MethodInfo info) { + if (info == null) { + return; + } + callOptions = callOptions.withOption(MethodInfo.KEY, info); + if (info.timeoutNanos != null) { + Deadline newDeadline = Deadline.after(info.timeoutNanos, TimeUnit.NANOSECONDS); + Deadline existingDeadline = callOptions.getDeadline(); + // If the new deadline is sooner than the existing deadline, swap them. + if (existingDeadline == null || newDeadline.compareTo(existingDeadline) < 0) { + callOptions = callOptions.withDeadline(newDeadline); + } + } + if (info.waitForReady != null) { + callOptions = + info.waitForReady ? callOptions.withWaitForReady() : callOptions.withoutWaitForReady(); + } + if (info.maxInboundMessageSize != null) { + Integer existingLimit = callOptions.getMaxInboundMessageSize(); + if (existingLimit != null) { + callOptions = + callOptions.withMaxInboundMessageSize( + Math.min(existingLimit, info.maxInboundMessageSize)); + } else { + callOptions = callOptions.withMaxInboundMessageSize(info.maxInboundMessageSize); + } + } + if (info.maxOutboundMessageSize != null) { + Integer existingLimit = callOptions.getMaxOutboundMessageSize(); + if (existingLimit != null) { + callOptions = + callOptions.withMaxOutboundMessageSize( + Math.min(existingLimit, info.maxOutboundMessageSize)); + } else { + callOptions = callOptions.withMaxOutboundMessageSize(info.maxOutboundMessageSize); + } + } + } + private static void logIfContextNarrowedTimeout( Deadline effectiveDeadline, @Nullable Deadline outerCallDeadline, @Nullable Deadline callDeadline) { diff --git a/core/src/main/java/io/grpc/internal/HedgingPolicy.java b/core/src/main/java/io/grpc/internal/HedgingPolicy.java index 83d1574961c..1b192141643 100644 --- a/core/src/main/java/io/grpc/internal/HedgingPolicy.java +++ b/core/src/main/java/io/grpc/internal/HedgingPolicy.java @@ -21,7 +21,6 @@ import com.google.common.collect.ImmutableSet; import io.grpc.Status.Code; import java.util.Set; -import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; /** @@ -70,18 +69,4 @@ public String toString() { .add("nonFatalStatusCodes", nonFatalStatusCodes) .toString(); } - - // TODO(zdapeng): delete this because HedgingPolicy will be always available prior to starting - // RetriableStream. - /** - * Provides the most suitable hedging policy for a call. - */ - interface Provider { - - /** - * This method is used no more than once for each call. - */ - @Nullable - HedgingPolicy get(); - } } diff --git a/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java b/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java index 2a46c1e31d2..b3a69bbce9b 100644 --- a/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java +++ b/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java @@ -22,8 +22,6 @@ import static io.grpc.ConnectivityState.IDLE; import static io.grpc.ConnectivityState.SHUTDOWN; import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; -import static io.grpc.internal.ServiceConfigInterceptor.HEDGING_POLICY_KEY; -import static io.grpc.internal.ServiceConfigInterceptor.RETRY_POLICY_KEY; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; @@ -74,6 +72,7 @@ import io.grpc.SynchronizationContext.ScheduledHandle; import io.grpc.internal.AutoConfiguredLoadBalancerFactory.AutoConfiguredLoadBalancer; import io.grpc.internal.ClientCallImpl.ClientStreamProvider; +import io.grpc.internal.ManagedChannelServiceConfig.MethodInfo; import io.grpc.internal.RetriableStream.ChannelBufferMeter; import io.grpc.internal.RetriableStream.Throttle; import java.net.URI; @@ -181,9 +180,6 @@ public void uncaughtException(Thread t, Throwable e) { private final long idleTimeoutMillis; private final ConnectivityStateManager channelStateManager = new ConnectivityStateManager(); - - private final ServiceConfigInterceptor serviceConfigInterceptor; - private final BackoffPolicy.Provider backoffPolicyProvider; /** @@ -529,6 +525,9 @@ public ClientStream newStream( } } else { final Throttle throttle = lastServiceConfig.getRetryThrottling(); + MethodInfo methodInfo = callOptions.getOption(MethodInfo.KEY); + final RetryPolicy retryPolicy = methodInfo == null ? null : methodInfo.retryPolicy; + final HedgingPolicy hedgingPolicy = methodInfo == null ? null : methodInfo.hedgingPolicy; final class RetryStream extends RetriableStream { @SuppressWarnings("unchecked") RetryStream() { @@ -540,8 +539,8 @@ final class RetryStream extends RetriableStream { channelBufferLimit, getCallExecutor(callOptions), transportFactory.getScheduledExecutorService(), - callOptions.getOption(RETRY_POLICY_KEY), - callOptions.getOption(HEDGING_POLICY_KEY), + retryPolicy, + hedgingPolicy, throttle); } @@ -640,7 +639,6 @@ public void execute(Runnable command) { this.delayedTransport.start(delayedTransportListener); this.backoffPolicyProvider = backoffPolicyProvider; - serviceConfigInterceptor = new ServiceConfigInterceptor(retryEnabled); if (builder.defaultServiceConfig != null) { ConfigOrError parsedDefaultServiceConfig = serviceConfigParser.parseServiceConfig(builder.defaultServiceConfig); @@ -656,7 +654,7 @@ public void execute(Runnable command) { } this.lookUpServiceConfig = builder.lookUpServiceConfig; realChannel = new RealChannel(nameResolver.getServiceAuthority()); - Channel channel = ClientInterceptors.intercept(realChannel, serviceConfigInterceptor); + Channel channel = realChannel; if (builder.binlog != null) { channel = builder.binlog.wrapChannel(channel); } @@ -701,18 +699,10 @@ public CallTracer create() { channelLogger.log( ChannelLogLevel.INFO, "Service config look-up disabled, using default service config"); } - handleServiceConfigUpdate(); + serviceConfigUpdated = true; } } - // May only be called in constructor or syncContext - private void handleServiceConfigUpdate() { - serviceConfigUpdated = true; - // TODO(zdapeng): get rid of serviceConfigInterceptor and do handle update together with - // configSelector. - serviceConfigInterceptor.handleUpdate(lastServiceConfig); - } - @VisibleForTesting static NameResolver getNameResolver(String target, NameResolver.Factory nameResolverFactory, NameResolver.Args nameResolverArgs) { @@ -1591,16 +1581,26 @@ public void run() { ChannelLogLevel.INFO, "Config selector from name resolver discarded by channel settings"); } - configSelector.set(null); + configSelector.set(effectiveServiceConfig.getDefaultConfigSelector()); } else { // Try to use config if returned from name resolver // Otherwise, try to use the default config if available if (validServiceConfig != null) { effectiveServiceConfig = validServiceConfig; - configSelector.set(resolvedConfigSelector); + if (resolvedConfigSelector != null) { + configSelector.set(resolvedConfigSelector); + if (effectiveServiceConfig.getDefaultConfigSelector() != null) { + channelLogger.log( + ChannelLogLevel.DEBUG, + "Method configs in service config will be discarded due to presence of" + + "config-selector"); + } + } else { + configSelector.set(effectiveServiceConfig.getDefaultConfigSelector()); + } } else if (defaultServiceConfig != null) { effectiveServiceConfig = defaultServiceConfig; - configSelector.set(null); + configSelector.set(effectiveServiceConfig.getDefaultConfigSelector()); channelLogger.log( ChannelLogLevel.INFO, "Received no service config, using default service config"); @@ -1631,7 +1631,7 @@ public void run() { // TODO(creamsoup): when `servers` is empty and lastResolutionStateCopy == SUCCESS // and lbNeedAddress, it shouldn't call the handleServiceConfigUpdate. But, // lbNeedAddress is not deterministic - handleServiceConfigUpdate(); + serviceConfigUpdated = true; } catch (RuntimeException re) { logger.log( Level.WARNING, diff --git a/core/src/main/java/io/grpc/internal/ManagedChannelServiceConfig.java b/core/src/main/java/io/grpc/internal/ManagedChannelServiceConfig.java index 4c64311ce89..08e9519c715 100644 --- a/core/src/main/java/io/grpc/internal/ManagedChannelServiceConfig.java +++ b/core/src/main/java/io/grpc/internal/ManagedChannelServiceConfig.java @@ -23,6 +23,9 @@ import com.google.common.base.MoreObjects; import com.google.common.base.Objects; import com.google.common.base.Strings; +import io.grpc.CallOptions; +import io.grpc.InternalConfigSelector; +import io.grpc.LoadBalancer.PickSubchannelArgs; import io.grpc.MethodDescriptor; import io.grpc.internal.RetriableStream.Throttle; import java.util.Collections; @@ -162,28 +165,30 @@ static ManagedChannelServiceConfig fromServiceConfig( healthCheckingConfig); } - /** - * Returns the per-service configuration for the channel. - */ - Map getServiceMap() { - return serviceMap; - } - @Nullable Map getHealthCheckingConfig() { return healthCheckingConfig; } /** - * Returns the per-method configuration for the channel. + * Used as a fallback per-RPC config supplier when the attributes value of {@link + * InternalConfigSelector#KEY} is not available. Returns {@code null} if there is no method + * config in this service config. */ - Map getServiceMethodMap() { - return serviceMethodMap; - } - @Nullable - MethodInfo getDefaultMethodConfig() { - return defaultMethodConfig; + InternalConfigSelector getDefaultConfigSelector() { + if (serviceMap.isEmpty() && serviceMethodMap.isEmpty() && defaultMethodConfig == null) { + return null; + } + return new InternalConfigSelector() { + @Override + public Result selectConfig(PickSubchannelArgs args) { + return Result.newBuilder() + .setConfig(ManagedChannelServiceConfig.this) + .setCallOptions(args.getCallOptions()) + .build(); + } + }; } @VisibleForTesting @@ -197,6 +202,19 @@ Throttle getRetryThrottling() { return retryThrottling; } + @Nullable + MethodInfo getMethodConfig(MethodDescriptor method) { + MethodInfo methodInfo = serviceMethodMap.get(method.getFullMethodName()); + if (methodInfo == null) { + String serviceName = method.getServiceName(); + methodInfo = serviceMap.get(serviceName); + } + if (methodInfo == null) { + methodInfo = defaultMethodConfig; + } + return methodInfo; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -231,6 +249,9 @@ public String toString() { * Equivalent of MethodConfig from a ServiceConfig with restrictions from Channel setting. */ static final class MethodInfo { + static final CallOptions.Key KEY = + CallOptions.Key.create("io.grpc.internal.ManagedChannelServiceConfig.MethodInfo"); + // TODO(carl-mastrangelo): add getters for these fields and make them private. final Long timeoutNanos; final Boolean waitForReady; diff --git a/core/src/main/java/io/grpc/internal/RetriableStream.java b/core/src/main/java/io/grpc/internal/RetriableStream.java index c4dccf04c53..4540538b40f 100644 --- a/core/src/main/java/io/grpc/internal/RetriableStream.java +++ b/core/src/main/java/io/grpc/internal/RetriableStream.java @@ -16,6 +16,7 @@ package io.grpc.internal; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; @@ -65,13 +66,11 @@ abstract class RetriableStream implements ClientStream { private final ScheduledExecutorService scheduledExecutorService; // Must not modify it. private final Metadata headers; - private final RetryPolicy.Provider retryPolicyProvider; - private final HedgingPolicy.Provider hedgingPolicyProvider; @Nullable - private RetryPolicy retryPolicy; + private final RetryPolicy retryPolicy; @Nullable - private HedgingPolicy hedgingPolicy; - private boolean isHedging; + private final HedgingPolicy hedgingPolicy; + private final boolean isHedging; /** Must be held when updating state, accessing state.buffer, or certain substream attributes. */ private final Object lock = new Object(); @@ -109,7 +108,7 @@ abstract class RetriableStream implements ClientStream { MethodDescriptor method, Metadata headers, ChannelBufferMeter channelBufferUsed, long perRpcBufferLimit, long channelBufferLimit, Executor callExecutor, ScheduledExecutorService scheduledExecutorService, - RetryPolicy.Provider retryPolicyProvider, HedgingPolicy.Provider hedgingPolicyProvider, + @Nullable RetryPolicy retryPolicy, @Nullable HedgingPolicy hedgingPolicy, @Nullable Throttle throttle) { this.method = method; this.channelBufferUsed = channelBufferUsed; @@ -118,8 +117,15 @@ abstract class RetriableStream implements ClientStream { this.callExecutor = callExecutor; this.scheduledExecutorService = scheduledExecutorService; this.headers = headers; - this.retryPolicyProvider = checkNotNull(retryPolicyProvider, "retryPolicyProvider"); - this.hedgingPolicyProvider = checkNotNull(hedgingPolicyProvider, "hedgingPolicyProvider"); + this.retryPolicy = retryPolicy; + if (retryPolicy != null) { + this.nextBackoffIntervalNanos = retryPolicy.initialBackoffNanos; + } + this.hedgingPolicy = hedgingPolicy; + checkArgument( + retryPolicy == null || hedgingPolicy == null, + "Should not provide both retryPolicy and hedgingPolicy"); + this.isHedging = hedgingPolicy != null; this.throttle = throttle; } @@ -316,10 +322,7 @@ public void runWith(Substream substream) { } Substream substream = createSubstream(0); - checkState(hedgingPolicy == null, "hedgingPolicy has been initialized unexpectedly"); - hedgingPolicy = hedgingPolicyProvider.get(); - if (null != hedgingPolicy) { - isHedging = true; + if (isHedging) { FutureCanceller scheduledHedgingRef = null; synchronized (lock) { @@ -806,9 +809,6 @@ public void closed(Status status, RpcProgress rpcProgress, Metadata trailers) { commitAndRun(newSubstream); } } else { - if (retryPolicy == null) { - retryPolicy = retryPolicyProvider.get(); - } if (retryPolicy == null || retryPolicy.maxAttempts == 1) { // optimization for early commit commitAndRun(newSubstream); @@ -895,12 +895,7 @@ public void run() { */ private RetryPlan makeRetryDecision(Status status, Metadata trailer) { if (retryPolicy == null) { - retryPolicy = retryPolicyProvider.get(); - if (retryPolicy == null) { - return new RetryPlan(false, 0); - } else { - nextBackoffIntervalNanos = retryPolicy.initialBackoffNanos; - } + return new RetryPlan(false, 0); } boolean shouldRetry = false; long backoffNanos = 0L; diff --git a/core/src/main/java/io/grpc/internal/RetryPolicy.java b/core/src/main/java/io/grpc/internal/RetryPolicy.java index c26e06d3376..36ab3a0ca27 100644 --- a/core/src/main/java/io/grpc/internal/RetryPolicy.java +++ b/core/src/main/java/io/grpc/internal/RetryPolicy.java @@ -22,7 +22,6 @@ import io.grpc.Status.Code; import java.util.Set; import javax.annotation.Nonnull; -import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; /** @@ -86,18 +85,4 @@ public String toString() { .add("retryableStatusCodes", retryableStatusCodes) .toString(); } - - // TODO(zdapeng): delete this because RetryPolicy will be always available prior to starting - // RetriableStream. - /** - * Provides the most suitable retry policy for a call. - */ - interface Provider { - - /** - * This method is used no more than once for each call. - */ - @Nullable - RetryPolicy get(); - } } diff --git a/core/src/main/java/io/grpc/internal/ServiceConfigInterceptor.java b/core/src/main/java/io/grpc/internal/ServiceConfigInterceptor.java deleted file mode 100644 index 4ffc25d5d4f..00000000000 --- a/core/src/main/java/io/grpc/internal/ServiceConfigInterceptor.java +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Copyright 2018 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.internal; - -import static com.google.common.base.Verify.verify; - -import com.google.common.annotations.VisibleForTesting; -import io.grpc.CallOptions; -import io.grpc.Channel; -import io.grpc.ClientCall; -import io.grpc.ClientInterceptor; -import io.grpc.Deadline; -import io.grpc.MethodDescriptor; -import io.grpc.internal.ManagedChannelServiceConfig.MethodInfo; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; -import javax.annotation.CheckForNull; -import javax.annotation.Nullable; - -/** - * Modifies RPCs in conformance with a Service Config. - */ -final class ServiceConfigInterceptor implements ClientInterceptor { - - // Map from method name to MethodInfo - @VisibleForTesting - final AtomicReference managedChannelServiceConfig = - new AtomicReference<>(); - - private final boolean retryEnabled; - - // Setting this to true and observing this equal to true are run in different threads. - private volatile boolean initComplete; - - ServiceConfigInterceptor(boolean retryEnabled) { - this.retryEnabled = retryEnabled; - } - - void handleUpdate(@Nullable ManagedChannelServiceConfig serviceConfig) { - managedChannelServiceConfig.set(serviceConfig); - initComplete = true; - } - - static final CallOptions.Key RETRY_POLICY_KEY = - CallOptions.Key.create("internal-retry-policy"); - static final CallOptions.Key HEDGING_POLICY_KEY = - CallOptions.Key.create("internal-hedging-policy"); - - @Override - public ClientCall interceptCall( - final MethodDescriptor method, CallOptions callOptions, Channel next) { - if (retryEnabled) { - if (initComplete) { - final RetryPolicy retryPolicy = getRetryPolicyFromConfig(method); - final class ImmediateRetryPolicyProvider implements RetryPolicy.Provider { - @Override - public RetryPolicy get() { - return retryPolicy; - } - } - - final HedgingPolicy hedgingPolicy = getHedgingPolicyFromConfig(method); - final class ImmediateHedgingPolicyProvider implements HedgingPolicy.Provider { - @Override - public HedgingPolicy get() { - return hedgingPolicy; - } - } - - verify( - retryPolicy == null || hedgingPolicy == null, - "Can not apply both retry and hedging policy for the method '%s'", method); - - callOptions = callOptions - .withOption(RETRY_POLICY_KEY, new ImmediateRetryPolicyProvider()) - .withOption(HEDGING_POLICY_KEY, new ImmediateHedgingPolicyProvider()); - } else { - final class DelayedRetryPolicyProvider implements RetryPolicy.Provider { - /** - * Returns RetryPolicy.DEFAULT if name resolving is not complete at the moment the method - * is invoked, otherwise returns the RetryPolicy computed from service config. - * - *

Note that this method is used no more than once for each call. - */ - @Override - @Nullable - public RetryPolicy get() { - if (!initComplete) { - return null; - } - return getRetryPolicyFromConfig(method); - } - } - - final class DelayedHedgingPolicyProvider implements HedgingPolicy.Provider { - /** - * Returns HedgingPolicy.DEFAULT if name resolving is not complete at the moment the - * method is invoked, otherwise returns the HedgingPolicy computed from service config. - * - *

Note that this method is used no more than once for each call. - */ - @Override - @Nullable - public HedgingPolicy get() { - if (!initComplete) { - return null; - } - HedgingPolicy hedgingPolicy = getHedgingPolicyFromConfig(method); - verify( - hedgingPolicy == null || getRetryPolicyFromConfig(method) == null, - "Can not apply both retry and hedging policy for the method '%s'", method); - return hedgingPolicy; - } - } - - callOptions = callOptions - .withOption(RETRY_POLICY_KEY, new DelayedRetryPolicyProvider()) - .withOption(HEDGING_POLICY_KEY, new DelayedHedgingPolicyProvider()); - } - } - - MethodInfo info = getMethodInfo(method); - if (info == null) { - return next.newCall(method, callOptions); - } - - if (info.timeoutNanos != null) { - Deadline newDeadline = Deadline.after(info.timeoutNanos, TimeUnit.NANOSECONDS); - Deadline existingDeadline = callOptions.getDeadline(); - // If the new deadline is sooner than the existing deadline, swap them. - if (existingDeadline == null || newDeadline.compareTo(existingDeadline) < 0) { - callOptions = callOptions.withDeadline(newDeadline); - } - } - if (info.waitForReady != null) { - callOptions = - info.waitForReady ? callOptions.withWaitForReady() : callOptions.withoutWaitForReady(); - } - if (info.maxInboundMessageSize != null) { - Integer existingLimit = callOptions.getMaxInboundMessageSize(); - if (existingLimit != null) { - callOptions = callOptions.withMaxInboundMessageSize( - Math.min(existingLimit, info.maxInboundMessageSize)); - } else { - callOptions = callOptions.withMaxInboundMessageSize(info.maxInboundMessageSize); - } - } - if (info.maxOutboundMessageSize != null) { - Integer existingLimit = callOptions.getMaxOutboundMessageSize(); - if (existingLimit != null) { - callOptions = callOptions.withMaxOutboundMessageSize( - Math.min(existingLimit, info.maxOutboundMessageSize)); - } else { - callOptions = callOptions.withMaxOutboundMessageSize(info.maxOutboundMessageSize); - } - } - - return next.newCall(method, callOptions); - } - - @CheckForNull - private MethodInfo getMethodInfo(MethodDescriptor method) { - ManagedChannelServiceConfig mcsc = managedChannelServiceConfig.get(); - if (mcsc == null) { - return null; - } - MethodInfo info; - info = mcsc.getServiceMethodMap().get(method.getFullMethodName()); - if (info == null) { - String serviceName = method.getServiceName(); - info = mcsc.getServiceMap().get(serviceName); - } - if (info == null) { - info = mcsc.getDefaultMethodConfig(); - } - return info; - } - - @VisibleForTesting - @Nullable - RetryPolicy getRetryPolicyFromConfig(MethodDescriptor method) { - MethodInfo info = getMethodInfo(method); - return info == null ? null : info.retryPolicy; - } - - @VisibleForTesting - @Nullable - HedgingPolicy getHedgingPolicyFromConfig(MethodDescriptor method) { - MethodInfo info = getMethodInfo(method); - return info == null ? null : info.hedgingPolicy; - } -} diff --git a/core/src/test/java/io/grpc/internal/ClientCallImplTest.java b/core/src/test/java/io/grpc/internal/ClientCallImplTest.java index 24c7d20db5a..a84bb225457 100644 --- a/core/src/test/java/io/grpc/internal/ClientCallImplTest.java +++ b/core/src/test/java/io/grpc/internal/ClientCallImplTest.java @@ -19,6 +19,8 @@ import static com.google.common.truth.Truth.assertThat; import static io.grpc.internal.ClientCallImpl.DEADLINE_EXPIRATION_CANCEL_DELAY_NANOS; import static io.grpc.internal.GrpcUtil.ACCEPT_ENCODING_SPLITTER; +import static java.util.concurrent.TimeUnit.MINUTES; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -38,6 +40,8 @@ import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; @@ -51,17 +55,20 @@ import io.grpc.Decompressor; import io.grpc.DecompressorRegistry; import io.grpc.InternalConfigSelector; +import io.grpc.LoadBalancer.PickSubchannelArgs; import io.grpc.Metadata; import io.grpc.MethodDescriptor; import io.grpc.MethodDescriptor.MethodType; import io.grpc.Status; import io.grpc.Status.Code; import io.grpc.internal.ClientCallImpl.ClientStreamProvider; +import io.grpc.internal.ManagedChannelServiceConfig.MethodInfo; import io.grpc.internal.testing.SingleMessageProducer; import io.grpc.testing.TestMethodDescriptors; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.util.Map; import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CountDownLatch; @@ -119,8 +126,7 @@ public class ClientCallImplTest { @Mock private ClientStream stream; - @Mock - private InternalConfigSelector configSelector; + private InternalConfigSelector configSelector = null; @Mock private ClientCall.Listener callListener; @@ -346,8 +352,8 @@ public void advertisedEncodingsAreSent() { call.start(callListener, new Metadata()); ArgumentCaptor metadataCaptor = ArgumentCaptor.forClass(Metadata.class); - verify(clientStreamProvider) - .newStream(eq(method), same(baseCallOptions), metadataCaptor.capture(), any(Context.class)); + verify(clientStreamProvider).newStream( + eq(method), same(baseCallOptions), metadataCaptor.capture(), any(Context.class)); Metadata actual = metadataCaptor.getValue(); // there should only be one. @@ -386,8 +392,153 @@ public void callOptionsPropagatedToTransport() { call.start(callListener, metadata); - verify(clientStreamProvider) - .newStream(same(method), same(callOptions), same(metadata), any(Context.class)); + verify(clientStreamProvider).newStream( + same(method), same(callOptions), same(metadata), any(Context.class)); + } + + @Test + public void configSelectorCallOptionsPropagatedToStream() { + ArgumentCaptor callOptionsCaptor = ArgumentCaptor.forClass(null); + configSelector = new InternalConfigSelector() { + @Override + public Result selectConfig(PickSubchannelArgs args) { + return Result.newBuilder() + .setConfig(ManagedChannelServiceConfig.empty()) + .setCallOptions(args.getCallOptions().withAuthority("dummy_value")) + .build(); + } + }; + ClientCallImpl call = new ClientCallImpl<>( + method, + MoreExecutors.directExecutor(), + baseCallOptions, + clientStreamProvider, + deadlineCancellationExecutor, + channelCallTracer, configSelector) + .setDecompressorRegistry(decompressorRegistry); + call.start(callListener, new Metadata()); + verify(clientStreamProvider).newStream( + same(method), callOptionsCaptor.capture(), any(Metadata.class), any(Context.class)); + assertThat(callOptionsCaptor.getValue().getAuthority()).isEqualTo("dummy_value"); + } + + @Test + public void methodConfigPropagatedToStream() { + Map rawMethodConfig = ImmutableMap.of( + "retryPolicy", + ImmutableMap.of( + "maxAttempts", 3.0D, + "initialBackoff", "1s", + "maxBackoff", "10s", + "backoffMultiplier", 1.5D, + "retryableStatusCodes", ImmutableList.of("UNAVAILABLE") + )); + final MethodInfo methodInfo = new MethodInfo(rawMethodConfig, true, 4, 4); + configSelector = new InternalConfigSelector() { + @Override + public Result selectConfig(PickSubchannelArgs args) { + ManagedChannelServiceConfig config = new ManagedChannelServiceConfig( + methodInfo, + ImmutableMap.of(), + ImmutableMap.of(), + null, + null, + null); + return Result.newBuilder() + .setConfig(config) + .setCallOptions(args.getCallOptions()) + .build(); + } + }; + ClientCallImpl call = new ClientCallImpl<>( + method, + MoreExecutors.directExecutor(), + baseCallOptions, + clientStreamProvider, + deadlineCancellationExecutor, + channelCallTracer, configSelector) + .setDecompressorRegistry(decompressorRegistry); + call.start(callListener, new Metadata()); + ArgumentCaptor callOptionsCaptor = ArgumentCaptor.forClass(null); + verify(clientStreamProvider).newStream( + same(method), callOptionsCaptor.capture(), any(Metadata.class), any(Context.class)); + assertThat(callOptionsCaptor.getValue().getOption(MethodInfo.KEY)).isEqualTo(methodInfo); + } + + @Test + public void configDeadlinePropagatedToStream() { + ArgumentCaptor callOptionsCaptor = ArgumentCaptor.forClass(null); + CallOptions callOptions = baseCallOptions.withDeadline(Deadline.after(2000, SECONDS)); + + // Case: config Deadline expires later than CallOptions Deadline + configSelector = new InternalConfigSelector() { + @Override + public Result selectConfig(PickSubchannelArgs args) { + Map rawMethodConfig = ImmutableMap.of( + "timeout", + "3000s"); + MethodInfo methodInfo = new MethodInfo(rawMethodConfig, false, 0, 0); + ManagedChannelServiceConfig config = new ManagedChannelServiceConfig( + methodInfo, + ImmutableMap.of(), + ImmutableMap.of(), + null, + null, + null); + return Result.newBuilder() + .setConfig(config) + .setCallOptions(args.getCallOptions()) + .build(); + } + }; + ClientCallImpl call = new ClientCallImpl<>( + method, + MoreExecutors.directExecutor(), + callOptions, + clientStreamProvider, + deadlineCancellationExecutor, + channelCallTracer, configSelector) + .setDecompressorRegistry(decompressorRegistry); + call.start(callListener, new Metadata()); + verify(clientStreamProvider).newStream( + same(method), callOptionsCaptor.capture(), any(Metadata.class), any(Context.class)); + Deadline actualDeadline = callOptionsCaptor.getValue().getDeadline(); + assertThat(actualDeadline).isLessThan(Deadline.after(2001, SECONDS)); + + // Case: config Deadline expires earlier than CallOptions Deadline + configSelector = new InternalConfigSelector() { + @Override + public Result selectConfig(PickSubchannelArgs args) { + Map rawMethodConfig = ImmutableMap.of( + "timeout", + "1000s"); + MethodInfo methodInfo = new MethodInfo(rawMethodConfig, false, 0, 0); + ManagedChannelServiceConfig config = new ManagedChannelServiceConfig( + methodInfo, + ImmutableMap.of(), + ImmutableMap.of(), + null, + null, + null); + return Result.newBuilder() + .setConfig(config) + .setCallOptions(args.getCallOptions()) + .build(); + } + }; + call = new ClientCallImpl<>( + method, + MoreExecutors.directExecutor(), + callOptions, + clientStreamProvider, + deadlineCancellationExecutor, + channelCallTracer, configSelector) + .setDecompressorRegistry(decompressorRegistry); + call.start(callListener, new Metadata()); + verify(clientStreamProvider, times(2)).newStream( + same(method), callOptionsCaptor.capture(), any(Metadata.class), any(Context.class)); + actualDeadline = callOptionsCaptor.getValue().getDeadline(); + assertThat(actualDeadline).isLessThan(Deadline.after(1001, MINUTES)); } @Test diff --git a/core/src/test/java/io/grpc/internal/HedgingPolicyTest.java b/core/src/test/java/io/grpc/internal/HedgingPolicyTest.java index 1a7cab8e70e..cdcfbded2e1 100644 --- a/core/src/test/java/io/grpc/internal/HedgingPolicyTest.java +++ b/core/src/test/java/io/grpc/internal/HedgingPolicyTest.java @@ -17,16 +17,9 @@ package io.grpc.internal; import static com.google.common.truth.Truth.assertThat; -import static io.grpc.internal.ServiceConfigInterceptor.HEDGING_POLICY_KEY; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; import com.google.common.collect.ImmutableSet; -import io.grpc.CallOptions; -import io.grpc.Channel; import io.grpc.MethodDescriptor; import io.grpc.Status.Code; import io.grpc.testing.TestMethodDescriptors; @@ -37,7 +30,6 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -import org.mockito.ArgumentCaptor; /** Unit tests for HedgingPolicy. */ @RunWith(JUnit4.class) @@ -58,54 +50,45 @@ public void getHedgingPolicies() throws Exception { @SuppressWarnings("unchecked") Map serviceConfig = (Map) serviceConfigObj; - - ServiceConfigInterceptor serviceConfigInterceptor = - new ServiceConfigInterceptor(/* retryEnabled= */ true); - serviceConfigInterceptor - .handleUpdate( - ManagedChannelServiceConfig - .fromServiceConfig( - serviceConfig, - /* retryEnabled= */ true, - /* maxRetryAttemptsLimit= */ 3, - /* maxHedgedAttemptsLimit= */ 4, - /* loadBalancingConfig= */ null)); + ManagedChannelServiceConfig channelServiceConfig = + ManagedChannelServiceConfig.fromServiceConfig( + serviceConfig, + /* retryEnabled= */ true, + /* maxRetryAttemptsLimit= */ 3, + /* maxHedgedAttemptsLimit= */ 4, + /* loadBalancingConfig= */ null); MethodDescriptor.Builder builder = TestMethodDescriptors.voidMethod().toBuilder(); MethodDescriptor method = builder.setFullMethodName("not/exist").build(); - assertThat(serviceConfigInterceptor.getHedgingPolicyFromConfig(method)).isNull(); + assertThat(channelServiceConfig.getMethodConfig(method)).isNull(); method = builder.setFullMethodName("not_exist/Foo1").build(); - assertThat(serviceConfigInterceptor.getHedgingPolicyFromConfig(method)).isNull(); + assertThat(channelServiceConfig.getMethodConfig(method)).isNull(); method = builder.setFullMethodName("SimpleService1/not_exist").build(); - - assertEquals( + assertThat(channelServiceConfig.getMethodConfig(method).hedgingPolicy).isEqualTo( new HedgingPolicy( 3, TimeUnit.MILLISECONDS.toNanos(2100), - ImmutableSet.of(Code.UNAVAILABLE, Code.RESOURCE_EXHAUSTED)), - serviceConfigInterceptor.getHedgingPolicyFromConfig(method)); + ImmutableSet.of(Code.UNAVAILABLE, Code.RESOURCE_EXHAUSTED))); method = builder.setFullMethodName("SimpleService1/Foo1").build(); - assertEquals( + assertThat(channelServiceConfig.getMethodConfig(method).hedgingPolicy).isEqualTo( new HedgingPolicy( 4, TimeUnit.MILLISECONDS.toNanos(100), - ImmutableSet.of(Code.UNAVAILABLE)), - serviceConfigInterceptor.getHedgingPolicyFromConfig(method)); + ImmutableSet.of(Code.UNAVAILABLE))); method = builder.setFullMethodName("SimpleService2/not_exist").build(); - assertThat(serviceConfigInterceptor.getHedgingPolicyFromConfig(method)).isNull(); + assertThat(channelServiceConfig.getMethodConfig(method).hedgingPolicy).isNull(); method = builder.setFullMethodName("SimpleService2/Foo2").build(); - assertEquals( + assertThat(channelServiceConfig.getMethodConfig(method).hedgingPolicy).isEqualTo( new HedgingPolicy( 4, TimeUnit.MILLISECONDS.toNanos(100), - ImmutableSet.of(Code.UNAVAILABLE)), - serviceConfigInterceptor.getHedgingPolicyFromConfig(method)); + ImmutableSet.of(Code.UNAVAILABLE))); } finally { if (reader != null) { reader.close(); @@ -115,8 +98,6 @@ public void getHedgingPolicies() throws Exception { @Test public void getRetryPolicies_hedgingDisabled() throws Exception { - Channel channel = mock(Channel.class); - ArgumentCaptor callOptionsCap = ArgumentCaptor.forClass(CallOptions.class); BufferedReader reader = null; try { reader = new BufferedReader(new InputStreamReader(RetryPolicyTest.class.getResourceAsStream( @@ -131,27 +112,18 @@ public void getRetryPolicies_hedgingDisabled() throws Exception { @SuppressWarnings("unchecked") Map serviceConfig = (Map) serviceConfigObj; - - ServiceConfigInterceptor serviceConfigInterceptor = - new ServiceConfigInterceptor(/* retryEnabled= */ false); - serviceConfigInterceptor - .handleUpdate( - ManagedChannelServiceConfig - .fromServiceConfig( - serviceConfig, - /* retryEnabled= */ false, - /* maxRetryAttemptsLimit= */ 3, - /* maxHedgedAttemptsLimit= */ 4, - /* loadBalancingConfig= */ null)); - + ManagedChannelServiceConfig channelServiceConfig = + ManagedChannelServiceConfig.fromServiceConfig( + serviceConfig, + /* retryEnabled= */ false, + /* maxRetryAttemptsLimit= */ 3, + /* maxHedgedAttemptsLimit= */ 4, + /* loadBalancingConfig= */ null); MethodDescriptor.Builder builder = TestMethodDescriptors.voidMethod().toBuilder(); MethodDescriptor method = builder.setFullMethodName("SimpleService1/Foo1").build(); - - serviceConfigInterceptor.interceptCall(method, CallOptions.DEFAULT, channel); - verify(channel).newCall(eq(method), callOptionsCap.capture()); - assertThat(callOptionsCap.getValue().getOption(HEDGING_POLICY_KEY)).isNull(); + assertThat(channelServiceConfig.getMethodConfig(method).hedgingPolicy).isNull(); } finally { if (reader != null) { reader.close(); diff --git a/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java b/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java index fdebd545ff0..cd14f409ca3 100644 --- a/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java +++ b/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java @@ -492,11 +492,23 @@ public void immediateDeadlineExceeded() { } @Test - public void startCallBeforeNameResolution() { + public void startCallBeforeNameResolution() throws Exception { + FakeNameResolverFactory nameResolverFactory = + new FakeNameResolverFactory.Builder(expectedUri) + .setServers(ImmutableList.of(addressGroup)).build(); + channelBuilder.nameResolverFactory(nameResolverFactory); channel = new ManagedChannelImpl( channelBuilder, mockTransportFactory, new FakeBackoffPolicyProvider(), balancerRpcExecutorPool, timer.getStopwatchSupplier(), Collections.emptyList(), timer.getTimeProvider()); + Map rawServiceConfig = + parseConfig("{\"methodConfig\":[{" + + "\"name\":[{\"service\":\"service\"}]," + + "\"waitForReady\":true}]}"); + ManagedChannelServiceConfig managedChannelServiceConfig = + createManagedChannelServiceConfig(rawServiceConfig, null); + nameResolverFactory.nextConfigOrError.set( + ConfigOrError.fromConfig(managedChannelServiceConfig)); Metadata headers = new Metadata(); ClientStream mockStream = mock(ClientStream.class); ClientCall call = channel.newCall(method, CallOptions.DEFAULT); @@ -526,8 +538,70 @@ channelBuilder, mockTransportFactory, new FakeBackoffPolicyProvider(), updateBalancingStateSafely(helper, READY, mockPicker); executor.runDueTasks(); - // TODO(zdapeng): verify CallOptions updated by config selector - verify(mockTransport).newStream(same(method), same(headers), any(CallOptions.class)); + ArgumentCaptor callOptionsCaptor = ArgumentCaptor.forClass(null); + verify(mockTransport).newStream(same(method), same(headers), callOptionsCaptor.capture()); + assertThat(callOptionsCaptor.getValue().isWaitForReady()).isTrue(); + verify(mockStream).start(streamListenerCaptor.capture()); + + // Clean up as much as possible to allow the channel to terminate. + shutdownSafely(helper, subchannel); + timer.forwardNanos( + TimeUnit.SECONDS.toNanos(ManagedChannelImpl.SUBCHANNEL_SHUTDOWN_DELAY_SECONDS)); + } + + @Test + public void newCallWithConfigSelector() { + FakeNameResolverFactory nameResolverFactory = + new FakeNameResolverFactory.Builder(expectedUri) + .setServers(ImmutableList.of(addressGroup)).build(); + channelBuilder.nameResolverFactory(nameResolverFactory); + channel = new ManagedChannelImpl( + channelBuilder, mockTransportFactory, new FakeBackoffPolicyProvider(), + balancerRpcExecutorPool, timer.getStopwatchSupplier(), + Collections.emptyList(), timer.getTimeProvider()); + nameResolverFactory.nextConfigOrError.set( + ConfigOrError.fromConfig(ManagedChannelServiceConfig.empty())); + InternalConfigSelector configSelector = new InternalConfigSelector() { + @Override + public Result selectConfig(PickSubchannelArgs args) { + return Result.newBuilder() + .setConfig(ManagedChannelServiceConfig.empty()) + .setCallOptions(args.getCallOptions().withAuthority("fake_override_authority")) + .build(); + } + }; + nameResolverFactory.nextAttributes.set( + Attributes.newBuilder().set(InternalConfigSelector.KEY, configSelector).build()); + channel.getState(true); + Metadata headers = new Metadata(); + ClientStream mockStream = mock(ClientStream.class); + ClientCall call = channel.newCall(method, CallOptions.DEFAULT); + call.start(mockCallListener, headers); + + ArgumentCaptor helperCaptor = ArgumentCaptor.forClass(null); + verify(mockLoadBalancerProvider).newLoadBalancer(helperCaptor.capture()); + helper = helperCaptor.getValue(); + // Make the transport available + Subchannel subchannel = + createSubchannelSafely(helper, addressGroup, Attributes.EMPTY, subchannelStateListener); + requestConnectionSafely(helper, subchannel); + verify(mockTransportFactory) + .newClientTransport( + any(SocketAddress.class), any(ClientTransportOptions.class), any(ChannelLogger.class)); + MockClientTransportInfo transportInfo = transports.poll(); + ConnectionClientTransport mockTransport = transportInfo.transport; + ManagedClientTransport.Listener transportListener = transportInfo.listener; + when(mockTransport.newStream(same(method), same(headers), any(CallOptions.class))) + .thenReturn(mockStream); + transportListener.transportReady(); + when(mockPicker.pickSubchannel(any(PickSubchannelArgs.class))) + .thenReturn(PickResult.withSubchannel(subchannel)); + updateBalancingStateSafely(helper, READY, mockPicker); + executor.runDueTasks(); + + ArgumentCaptor callOptionsCaptor = ArgumentCaptor.forClass(null); + verify(mockTransport).newStream(same(method), same(headers), callOptionsCaptor.capture()); + assertThat(callOptionsCaptor.getValue().getAuthority()).isEqualTo("fake_override_authority"); verify(mockStream).start(streamListenerCaptor.capture()); // Clean up as much as possible to allow the channel to terminate. @@ -3704,8 +3778,8 @@ public void nameResolverHelper_emptyConfigSucceeds() { assertThat(coe.getError()).isNull(); ManagedChannelServiceConfig cfg = (ManagedChannelServiceConfig) coe.getConfig(); - assertThat(cfg.getServiceMap()).isEmpty(); - assertThat(cfg.getServiceMethodMap()).isEmpty(); + assertThat(cfg.getMethodConfig(method)).isEqualTo( + ManagedChannelServiceConfig.empty().getMethodConfig(method)); } @Test diff --git a/core/src/test/java/io/grpc/internal/ManagedChannelServiceConfigTest.java b/core/src/test/java/io/grpc/internal/ManagedChannelServiceConfigTest.java index 5a7bad5a1b7..dde3adf84d6 100644 --- a/core/src/test/java/io/grpc/internal/ManagedChannelServiceConfigTest.java +++ b/core/src/test/java/io/grpc/internal/ManagedChannelServiceConfigTest.java @@ -17,9 +17,19 @@ package io.grpc.internal; import static com.google.common.truth.Truth.assertThat; +import static io.grpc.MethodDescriptor.MethodType.UNARY; +import static io.grpc.Status.Code.UNAVAILABLE; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.grpc.CallOptions; +import io.grpc.InternalConfigSelector; +import io.grpc.InternalConfigSelector.Result; +import io.grpc.Metadata; +import io.grpc.MethodDescriptor; +import io.grpc.internal.ManagedChannelServiceConfig.MethodInfo; +import io.grpc.testing.TestMethodDescriptors; import java.util.Collections; import java.util.Map; import org.junit.Rule; @@ -136,6 +146,80 @@ public void createManagedChannelServiceConfig_failsOnMissingServiceName() { ManagedChannelServiceConfig.fromServiceConfig(serviceConfig, true, 3, 4, null); } + @Test + public void managedChannelServiceConfig_parseMethodConfig() { + Map name1 = ImmutableMap.of("service", "service1", "method", "method1"); + Map name2 = ImmutableMap.of("service", "service2"); + Map methodConfig = ImmutableMap.of( + "name", ImmutableList.of(name1, name2), + "timeout", "1.234s", + "retryPolicy", + ImmutableMap.of( + "maxAttempts", 3.0D, + "initialBackoff", "1s", + "maxBackoff", "10s", + "backoffMultiplier", 1.5D, + "retryableStatusCodes", ImmutableList.of("UNAVAILABLE") + )); + Map defaultMethodConfig = ImmutableMap.of( + "name", ImmutableList.of(ImmutableMap.of()), + "timeout", "4.321s"); + Map rawServiceConfig = + ImmutableMap.of("methodConfig", ImmutableList.of(methodConfig, defaultMethodConfig)); + + // retry disabled + ManagedChannelServiceConfig serviceConfig = + ManagedChannelServiceConfig.fromServiceConfig(rawServiceConfig, false, 0, 0, null); + MethodInfo methodInfo = serviceConfig.getMethodConfig(methodForName("service1", "method1")); + assertThat(methodInfo.timeoutNanos).isEqualTo(MILLISECONDS.toNanos(1234)); + assertThat(methodInfo.retryPolicy).isNull(); + methodInfo = serviceConfig.getMethodConfig(methodForName("service1", "methodX")); + assertThat(methodInfo.timeoutNanos).isEqualTo(MILLISECONDS.toNanos(4321)); + assertThat(methodInfo.retryPolicy).isNull(); + methodInfo = serviceConfig.getMethodConfig(methodForName("service2", "methodX")); + assertThat(methodInfo.timeoutNanos).isEqualTo(MILLISECONDS.toNanos(1234)); + assertThat(methodInfo.retryPolicy).isNull(); + + // retry enabled + serviceConfig = + ManagedChannelServiceConfig.fromServiceConfig(rawServiceConfig, true, 2, 0, null); + methodInfo = serviceConfig.getMethodConfig(methodForName("service1", "method1")); + assertThat(methodInfo.timeoutNanos).isEqualTo(MILLISECONDS.toNanos(1234)); + assertThat(methodInfo.retryPolicy.maxAttempts).isEqualTo(2); + assertThat(methodInfo.retryPolicy.retryableStatusCodes).containsExactly(UNAVAILABLE); + methodInfo = serviceConfig.getMethodConfig(methodForName("service1", "methodX")); + assertThat(methodInfo.timeoutNanos).isEqualTo(MILLISECONDS.toNanos(4321)); + assertThat(methodInfo.retryPolicy).isNull(); + } + + @Test + public void getDefaultConfigSelectorFromConfig() { + Map name = ImmutableMap.of("service", "service1", "method", "method1"); + Map methodConfig = ImmutableMap.of( + "name", ImmutableList.of(name), "timeout", "1.234s"); + Map rawServiceConfig = + ImmutableMap.of("methodConfig", ImmutableList.of(methodConfig)); + ManagedChannelServiceConfig serviceConfig = + ManagedChannelServiceConfig.fromServiceConfig(rawServiceConfig, false, 0, 0, null); + InternalConfigSelector configSelector = serviceConfig.getDefaultConfigSelector(); + MethodDescriptor method = methodForName("service1", "method1"); + Result result = configSelector.selectConfig( + new PickSubchannelArgsImpl(method, new Metadata(), CallOptions.DEFAULT)); + MethodInfo methodInfoFromDefaultConfigSelector = + ((ManagedChannelServiceConfig) result.getConfig()).getMethodConfig(method); + assertThat(methodInfoFromDefaultConfigSelector) + .isEqualTo(serviceConfig.getMethodConfig(method)); + } + + private static MethodDescriptor methodForName(String service, String method) { + return MethodDescriptor.newBuilder() + .setFullMethodName(service + "/" + method) + .setRequestMarshaller(TestMethodDescriptors.voidMarshaller()) + .setResponseMarshaller(TestMethodDescriptors.voidMarshaller()) + .setType(UNARY) + .build(); + } + @SuppressWarnings("unchecked") private static Map parseConfig(String json) throws Exception { return (Map) JsonParser.parse(json); diff --git a/core/src/test/java/io/grpc/internal/RetriableStreamTest.java b/core/src/test/java/io/grpc/internal/RetriableStreamTest.java index 5aab2f0856a..88812b17902 100644 --- a/core/src/test/java/io/grpc/internal/RetriableStreamTest.java +++ b/core/src/test/java/io/grpc/internal/RetriableStreamTest.java @@ -144,24 +144,14 @@ private final class RecordedRetriableStream extends RetriableStream { ChannelBufferMeter channelBufferUsed, long perRpcBufferLimit, long channelBufferLimit, Executor callExecutor, ScheduledExecutorService scheduledExecutorService, - @Nullable final RetryPolicy retryPolicy, - @Nullable final HedgingPolicy hedgingPolicy, + @Nullable RetryPolicy retryPolicy, + @Nullable HedgingPolicy hedgingPolicy, @Nullable Throttle throttle) { super( method, headers, channelBufferUsed, perRpcBufferLimit, channelBufferLimit, callExecutor, scheduledExecutorService, - new RetryPolicy.Provider() { - @Override - public RetryPolicy get() { - return retryPolicy; - } - }, - new HedgingPolicy.Provider() { - @Override - public HedgingPolicy get() { - return hedgingPolicy; - } - }, + retryPolicy, + hedgingPolicy, throttle); } diff --git a/core/src/test/java/io/grpc/internal/RetryPolicyTest.java b/core/src/test/java/io/grpc/internal/RetryPolicyTest.java index aab50df7500..5ff96df539b 100644 --- a/core/src/test/java/io/grpc/internal/RetryPolicyTest.java +++ b/core/src/test/java/io/grpc/internal/RetryPolicyTest.java @@ -17,17 +17,11 @@ package io.grpc.internal; import static com.google.common.truth.Truth.assertThat; -import static io.grpc.internal.ServiceConfigInterceptor.RETRY_POLICY_KEY; import static java.lang.Double.parseDouble; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; import com.google.common.collect.ImmutableSet; -import io.grpc.CallOptions; -import io.grpc.Channel; import io.grpc.MethodDescriptor; import io.grpc.Status.Code; import io.grpc.internal.RetriableStream.Throttle; @@ -39,7 +33,6 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -import org.mockito.ArgumentCaptor; /** Unit tests for RetryPolicy. */ @RunWith(JUnit4.class) @@ -62,59 +55,51 @@ public void getRetryPolicies() throws Exception { @SuppressWarnings("unchecked") Map serviceConfig = (Map) serviceConfigObj; - ServiceConfigInterceptor serviceConfigInterceptor = - new ServiceConfigInterceptor(/* retryEnabled= */ true); - serviceConfigInterceptor - .handleUpdate( - ManagedChannelServiceConfig - .fromServiceConfig( - serviceConfig, - /* retryEnabled= */ true, - /* maxRetryAttemptsLimit= */ 4, - /* maxHedgedAttemptsLimit= */ 3, - /* loadBalancingConfig= */ null)); + ManagedChannelServiceConfig channelServiceConfig = + ManagedChannelServiceConfig.fromServiceConfig( + serviceConfig, + /* retryEnabled= */ true, + /* maxRetryAttemptsLimit= */ 4, + /* maxHedgedAttemptsLimit= */ 3, + /* loadBalancingConfig= */ null); MethodDescriptor.Builder builder = TestMethodDescriptors.voidMethod().toBuilder(); MethodDescriptor method = builder.setFullMethodName("not/exist").build(); - assertThat(serviceConfigInterceptor.getRetryPolicyFromConfig(method)).isNull(); + assertThat(channelServiceConfig.getMethodConfig(method)).isNull(); method = builder.setFullMethodName("not_exist/Foo1").build(); - assertThat(serviceConfigInterceptor.getRetryPolicyFromConfig(method)).isNull(); + assertThat(channelServiceConfig.getMethodConfig(method)).isNull(); method = builder.setFullMethodName("SimpleService1/not_exist").build(); - - assertEquals( + assertThat(channelServiceConfig.getMethodConfig(method).retryPolicy).isEqualTo( new RetryPolicy( 3, TimeUnit.MILLISECONDS.toNanos(2100), TimeUnit.MILLISECONDS.toNanos(2200), parseDouble("3"), - ImmutableSet.of(Code.UNAVAILABLE, Code.RESOURCE_EXHAUSTED)), - serviceConfigInterceptor.getRetryPolicyFromConfig(method)); + ImmutableSet.of(Code.UNAVAILABLE, Code.RESOURCE_EXHAUSTED))); method = builder.setFullMethodName("SimpleService1/Foo1").build(); - assertEquals( + assertThat(channelServiceConfig.getMethodConfig(method).retryPolicy).isEqualTo( new RetryPolicy( 4, TimeUnit.MILLISECONDS.toNanos(100), TimeUnit.MILLISECONDS.toNanos(1000), parseDouble("2"), - ImmutableSet.of(Code.UNAVAILABLE)), - serviceConfigInterceptor.getRetryPolicyFromConfig(method)); + ImmutableSet.of(Code.UNAVAILABLE))); method = builder.setFullMethodName("SimpleService2/not_exist").build(); - assertThat(serviceConfigInterceptor.getRetryPolicyFromConfig(method)).isNull(); + assertThat(channelServiceConfig.getMethodConfig(method).retryPolicy).isNull(); method = builder.setFullMethodName("SimpleService2/Foo2").build(); - assertEquals( + assertThat(channelServiceConfig.getMethodConfig(method).retryPolicy).isEqualTo( new RetryPolicy( 4, TimeUnit.MILLISECONDS.toNanos(100), TimeUnit.MILLISECONDS.toNanos(1000), parseDouble("2"), - ImmutableSet.of(Code.UNAVAILABLE)), - serviceConfigInterceptor.getRetryPolicyFromConfig(method)); + ImmutableSet.of(Code.UNAVAILABLE))); } finally { if (reader != null) { reader.close(); @@ -124,8 +109,6 @@ public void getRetryPolicies() throws Exception { @Test public void getRetryPolicies_retryDisabled() throws Exception { - Channel channel = mock(Channel.class); - ArgumentCaptor callOptionsCap = ArgumentCaptor.forClass(CallOptions.class); BufferedReader reader = null; try { reader = new BufferedReader(new InputStreamReader(RetryPolicyTest.class.getResourceAsStream( @@ -140,27 +123,17 @@ public void getRetryPolicies_retryDisabled() throws Exception { @SuppressWarnings("unchecked") Map serviceConfig = (Map) serviceConfigObj; - - ServiceConfigInterceptor serviceConfigInterceptor = - new ServiceConfigInterceptor(/* retryEnabled= */ false); - serviceConfigInterceptor - .handleUpdate( - ManagedChannelServiceConfig - .fromServiceConfig( - serviceConfig, - /* retryEnabled= */ false, - /* maxRetryAttemptsLimit= */ 4, - /* maxHedgedAttemptsLimit= */ 3, - /* loadBalancingConfig= */ null)); - + ManagedChannelServiceConfig channelServiceConfig = + ManagedChannelServiceConfig.fromServiceConfig( + serviceConfig, + /* retryEnabled= */ false, + /* maxRetryAttemptsLimit= */ 4, + /* maxHedgedAttemptsLimit= */ 3, + /* loadBalancingConfig= */ null); MethodDescriptor.Builder builder = TestMethodDescriptors.voidMethod().toBuilder(); - MethodDescriptor method = builder.setFullMethodName("SimpleService1/Foo1").build(); - - serviceConfigInterceptor.interceptCall(method, CallOptions.DEFAULT, channel); - verify(channel).newCall(eq(method), callOptionsCap.capture()); - assertThat(callOptionsCap.getValue().getOption(RETRY_POLICY_KEY)).isNull(); + assertThat(channelServiceConfig.getMethodConfig(method).retryPolicy).isNull(); } finally { if (reader != null) { reader.close(); diff --git a/core/src/test/java/io/grpc/internal/ServiceConfigInterceptorTest.java b/core/src/test/java/io/grpc/internal/ServiceConfigInterceptorTest.java deleted file mode 100644 index 4a61587c312..00000000000 --- a/core/src/test/java/io/grpc/internal/ServiceConfigInterceptorTest.java +++ /dev/null @@ -1,528 +0,0 @@ -/* - * Copyright 2018 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.internal; - -import static com.google.common.truth.Truth.assertThat; -import static io.grpc.internal.ServiceConfigInterceptor.HEDGING_POLICY_KEY; -import static io.grpc.internal.ServiceConfigInterceptor.RETRY_POLICY_KEY; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.verify; - -import io.grpc.CallOptions; -import io.grpc.Channel; -import io.grpc.Deadline; -import io.grpc.MethodDescriptor; -import io.grpc.MethodDescriptor.MethodType; -import io.grpc.internal.ManagedChannelServiceConfig.MethodInfo; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.concurrent.TimeUnit; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; - -/** - * Unit tests for {@link ServiceConfigInterceptor}. - */ -@RunWith(JUnit4.class) -public class ServiceConfigInterceptorTest { - - @Rule public final ExpectedException thrown = ExpectedException.none(); - - @Mock private Channel channel; - @Captor private ArgumentCaptor callOptionsCap; - - @Before - public void setUp() { - MockitoAnnotations.initMocks(this); - } - - private final ServiceConfigInterceptor interceptor = - new ServiceConfigInterceptor(/* retryEnabled = */ true); - - private final String fullMethodName = - MethodDescriptor.generateFullMethodName("service", "method"); - private final MethodDescriptor methodDescriptor = - MethodDescriptor.newBuilder(new NoopMarshaller(), new NoopMarshaller()) - .setType(MethodType.UNARY) - .setFullMethodName(fullMethodName) - .build(); - - private static final class JsonObj extends HashMap { - private JsonObj(Object ... kv) { - for (int i = 0; i < kv.length; i += 2) { - put((String) kv[i], kv[i + 1]); - } - } - } - - private static final class JsonList extends ArrayList { - private JsonList(Object ... values) { - addAll(Arrays.asList(values)); - } - } - - @Test - public void withWaitForReady() { - JsonObj name = new JsonObj("service", "service"); - JsonObj methodConfig = new JsonObj("name", new JsonList(name), "waitForReady", true); - JsonObj serviceConfig = new JsonObj("methodConfig", new JsonList(methodConfig)); - ManagedChannelServiceConfig parsedServiceConfig = - createManagedChannelServiceConfig(serviceConfig); - - interceptor.handleUpdate(parsedServiceConfig); - - interceptor.interceptCall(methodDescriptor, CallOptions.DEFAULT.withoutWaitForReady(), channel); - - verify(channel).newCall(eq(methodDescriptor), callOptionsCap.capture()); - assertThat(callOptionsCap.getValue().isWaitForReady()).isTrue(); - } - - @Test - public void handleNullConfig() { - JsonObj name = new JsonObj("service", "service"); - JsonObj methodConfig = new JsonObj("name", new JsonList(name), "waitForReady", true); - JsonObj serviceConfig = new JsonObj("methodConfig", new JsonList(methodConfig)); - - interceptor.handleUpdate(createManagedChannelServiceConfig(serviceConfig)); - interceptor.handleUpdate(null); - - interceptor.interceptCall(methodDescriptor, CallOptions.DEFAULT.withoutWaitForReady(), channel); - - verify(channel).newCall(eq(methodDescriptor), callOptionsCap.capture()); - assertThat(callOptionsCap.getValue().isWaitForReady()).isFalse(); - } - - @Test - public void handleUpdateNotCalledBeforeInterceptCall() { - interceptor.interceptCall(methodDescriptor, CallOptions.DEFAULT.withoutWaitForReady(), channel); - - verify(channel).newCall(eq(methodDescriptor), callOptionsCap.capture()); - assertThat(callOptionsCap.getValue().isWaitForReady()).isFalse(); - assertThat(callOptionsCap.getValue().getOption(RETRY_POLICY_KEY).get()) - .isNull(); - assertThat(callOptionsCap.getValue().getOption(HEDGING_POLICY_KEY).get()) - .isNull(); - } - - @Test - public void withMaxRequestSize() { - JsonObj name = new JsonObj("service", "service"); - JsonObj methodConfig = new JsonObj("name", new JsonList(name), "maxRequestMessageBytes", 1d); - JsonObj serviceConfig = new JsonObj("methodConfig", new JsonList(methodConfig)); - ManagedChannelServiceConfig parsedServiceConfig = - createManagedChannelServiceConfig(serviceConfig); - - interceptor.handleUpdate(parsedServiceConfig); - - interceptor.interceptCall(methodDescriptor, CallOptions.DEFAULT, channel); - - verify(channel).newCall(eq(methodDescriptor), callOptionsCap.capture()); - assertThat(callOptionsCap.getValue().getMaxOutboundMessageSize()).isEqualTo(1); - } - - @Test - public void withMaxRequestSize_pickSmallerExisting() { - JsonObj name = new JsonObj("service", "service"); - JsonObj methodConfig = new JsonObj("name", new JsonList(name), "maxRequestMessageBytes", 10d); - JsonObj serviceConfig = new JsonObj("methodConfig", new JsonList(methodConfig)); - ManagedChannelServiceConfig parsedServiceConfig = - createManagedChannelServiceConfig(serviceConfig); - - interceptor.handleUpdate(parsedServiceConfig); - - interceptor.interceptCall( - methodDescriptor, CallOptions.DEFAULT.withMaxOutboundMessageSize(5), channel); - - verify(channel).newCall(eq(methodDescriptor), callOptionsCap.capture()); - assertThat(callOptionsCap.getValue().getMaxOutboundMessageSize()).isEqualTo(5); - } - - @Test - public void withMaxRequestSize_pickSmallerNew() { - JsonObj name = new JsonObj("service", "service"); - JsonObj methodConfig = new JsonObj("name", new JsonList(name), "maxRequestMessageBytes", 5d); - JsonObj serviceConfig = new JsonObj("methodConfig", new JsonList(methodConfig)); - ManagedChannelServiceConfig parsedServiceConfig = - createManagedChannelServiceConfig(serviceConfig); - - interceptor.handleUpdate(parsedServiceConfig); - - interceptor.interceptCall( - methodDescriptor, CallOptions.DEFAULT.withMaxOutboundMessageSize(10), channel); - - verify(channel).newCall(eq(methodDescriptor), callOptionsCap.capture()); - assertThat(callOptionsCap.getValue().getMaxOutboundMessageSize()).isEqualTo(5); - } - - @Test - public void withMaxResponseSize() { - JsonObj name = new JsonObj("service", "service"); - JsonObj methodConfig = new JsonObj("name", new JsonList(name), "maxResponseMessageBytes", 1d); - JsonObj serviceConfig = new JsonObj("methodConfig", new JsonList(methodConfig)); - ManagedChannelServiceConfig parsedServiceConfig = - createManagedChannelServiceConfig(serviceConfig); - - interceptor.handleUpdate(parsedServiceConfig); - - interceptor.interceptCall(methodDescriptor, CallOptions.DEFAULT, channel); - - verify(channel).newCall(eq(methodDescriptor), callOptionsCap.capture()); - assertThat(callOptionsCap.getValue().getMaxInboundMessageSize()).isEqualTo(1); - } - - @Test - public void withMaxResponseSize_pickSmallerExisting() { - JsonObj name = new JsonObj("service", "service"); - JsonObj methodConfig = new JsonObj("name", new JsonList(name), "maxResponseMessageBytes", 5d); - JsonObj serviceConfig = new JsonObj("methodConfig", new JsonList(methodConfig)); - ManagedChannelServiceConfig parsedServiceConfig = - createManagedChannelServiceConfig(serviceConfig); - - interceptor.handleUpdate(parsedServiceConfig); - - interceptor.interceptCall( - methodDescriptor, CallOptions.DEFAULT.withMaxInboundMessageSize(10), channel); - - verify(channel).newCall(eq(methodDescriptor), callOptionsCap.capture()); - assertThat(callOptionsCap.getValue().getMaxInboundMessageSize()).isEqualTo(5); - } - - @Test - public void withMaxResponseSize_pickSmallerNew() { - JsonObj name = new JsonObj("service", "service"); - JsonObj methodConfig = new JsonObj("name", new JsonList(name), "maxResponseMessageBytes", 10d); - JsonObj serviceConfig = new JsonObj("methodConfig", new JsonList(methodConfig)); - ManagedChannelServiceConfig parsedServiceConfig = - createManagedChannelServiceConfig(serviceConfig); - - interceptor.handleUpdate(parsedServiceConfig); - - interceptor.interceptCall( - methodDescriptor, CallOptions.DEFAULT.withMaxInboundMessageSize(5), channel); - - verify(channel).newCall(eq(methodDescriptor), callOptionsCap.capture()); - assertThat(callOptionsCap.getValue().getMaxInboundMessageSize()).isEqualTo(5); - } - - @Test - public void withoutWaitForReady() { - JsonObj name = new JsonObj("service", "service"); - JsonObj methodConfig = new JsonObj("name", new JsonList(name), "waitForReady", false); - JsonObj serviceConfig = new JsonObj("methodConfig", new JsonList(methodConfig)); - ManagedChannelServiceConfig parsedServiceConfig = - createManagedChannelServiceConfig(serviceConfig); - - interceptor.handleUpdate(parsedServiceConfig); - - interceptor.interceptCall(methodDescriptor, CallOptions.DEFAULT.withWaitForReady(), channel); - - verify(channel).newCall(eq(methodDescriptor), callOptionsCap.capture()); - assertThat(callOptionsCap.getValue().isWaitForReady()).isFalse(); - } - - @Test - public void fullMethodMatched() { - // Put in service that matches, but has no deadline. It should be lower priority - JsonObj name1 = new JsonObj("service", "service"); - JsonObj methodConfig1 = new JsonObj("name", new JsonList(name1)); - - JsonObj name2 = new JsonObj("service", "service", "method", "method"); - JsonObj methodConfig2 = new JsonObj("name", new JsonList(name2), "timeout", "1s"); - - JsonObj serviceConfig = new JsonObj("methodConfig", new JsonList(methodConfig1, methodConfig2)); - ManagedChannelServiceConfig parsedServiceConfig = - createManagedChannelServiceConfig(serviceConfig); - - interceptor.handleUpdate(parsedServiceConfig); - - interceptor.interceptCall(methodDescriptor, CallOptions.DEFAULT, channel); - - verify(channel).newCall(eq(methodDescriptor), callOptionsCap.capture()); - assertThat(callOptionsCap.getValue().getDeadline()).isNotNull(); - } - - @Test - public void nearerDeadlineKept_existing() { - JsonObj name = new JsonObj("service", "service"); - JsonObj methodConfig = new JsonObj("name", new JsonList(name), "timeout", "100000s"); - JsonObj serviceConfig = new JsonObj("methodConfig", new JsonList(methodConfig)); - ManagedChannelServiceConfig parsedServiceConfig = - createManagedChannelServiceConfig(serviceConfig); - - interceptor.handleUpdate(parsedServiceConfig); - - Deadline existingDeadline = Deadline.after(1000, TimeUnit.NANOSECONDS); - interceptor.interceptCall( - methodDescriptor, CallOptions.DEFAULT.withDeadline(existingDeadline), channel); - - verify(channel).newCall(eq(methodDescriptor), callOptionsCap.capture()); - assertThat(callOptionsCap.getValue().getDeadline()).isEqualTo(existingDeadline); - } - - @Test - public void nearerDeadlineKept_new() { - // TODO(carl-mastrangelo): the deadlines are very large because they change over time. - // This should be fixed, and is tracked in https://github.com/grpc/grpc-java/issues/2531 - JsonObj name = new JsonObj("service", "service"); - JsonObj methodConfig = new JsonObj("name", new JsonList(name), "timeout", "1s"); - JsonObj serviceConfig = new JsonObj("methodConfig", new JsonList(methodConfig)); - ManagedChannelServiceConfig parsedServiceConfig = - createManagedChannelServiceConfig(serviceConfig); - - interceptor.handleUpdate(parsedServiceConfig); - - Deadline existingDeadline = Deadline.after(1234567890, TimeUnit.NANOSECONDS); - interceptor.interceptCall( - methodDescriptor, CallOptions.DEFAULT.withDeadline(existingDeadline), channel); - - verify(channel).newCall(eq(methodDescriptor), callOptionsCap.capture()); - assertThat(callOptionsCap.getValue().getDeadline()).isNotEqualTo(existingDeadline); - } - - @Test - public void handleUpdate_onEmptyName() { - JsonObj methodConfig = new JsonObj(); - JsonObj serviceConfig = new JsonObj("methodConfig", new JsonList(methodConfig)); - ManagedChannelServiceConfig parsedServiceConfig = - createManagedChannelServiceConfig(serviceConfig); - - interceptor.handleUpdate(parsedServiceConfig); - - assertThat(interceptor.managedChannelServiceConfig.get().getDefaultMethodConfig()).isNull(); - assertThat(interceptor.managedChannelServiceConfig.get().getServiceMap()).isEmpty(); - assertThat(interceptor.managedChannelServiceConfig.get().getServiceMethodMap()).isEmpty(); - } - - @Test - public void handleUpdate_onDefaultMethodConfig() { - JsonObj name = new JsonObj(); - JsonObj methodConfig = new JsonObj("name", new JsonList(name)); - JsonObj serviceConfig = new JsonObj("methodConfig", new JsonList(methodConfig)); - ManagedChannelServiceConfig parsedServiceConfig = - createManagedChannelServiceConfig(serviceConfig); - interceptor.handleUpdate(parsedServiceConfig); - assertThat(interceptor.managedChannelServiceConfig.get().getDefaultMethodConfig()) - .isEqualTo(new MethodInfo(methodConfig, false, 1, 1)); - assertThat(interceptor.managedChannelServiceConfig.get().getServiceMap()).isEmpty(); - assertThat(interceptor.managedChannelServiceConfig.get().getServiceMethodMap()).isEmpty(); - - name = new JsonObj("method", ""); - methodConfig = new JsonObj("name", new JsonList(name)); - serviceConfig = new JsonObj("methodConfig", new JsonList(methodConfig)); - parsedServiceConfig = createManagedChannelServiceConfig(serviceConfig); - interceptor.handleUpdate(parsedServiceConfig); - assertThat(interceptor.managedChannelServiceConfig.get().getDefaultMethodConfig()) - .isEqualTo(new MethodInfo(methodConfig, false, 1, 1)); - assertThat(interceptor.managedChannelServiceConfig.get().getServiceMap()).isEmpty(); - assertThat(interceptor.managedChannelServiceConfig.get().getServiceMethodMap()).isEmpty(); - - name = new JsonObj("service", ""); - methodConfig = new JsonObj("name", new JsonList(name)); - serviceConfig = new JsonObj("methodConfig", new JsonList(methodConfig)); - parsedServiceConfig = createManagedChannelServiceConfig(serviceConfig); - interceptor.handleUpdate(parsedServiceConfig); - assertThat(interceptor.managedChannelServiceConfig.get().getDefaultMethodConfig()) - .isEqualTo(new MethodInfo(methodConfig, false, 1, 1)); - assertThat(interceptor.managedChannelServiceConfig.get().getServiceMap()).isEmpty(); - assertThat(interceptor.managedChannelServiceConfig.get().getServiceMethodMap()).isEmpty(); - - name = new JsonObj("service", "", "method", ""); - methodConfig = new JsonObj("name", new JsonList(name)); - serviceConfig = new JsonObj("methodConfig", new JsonList(methodConfig)); - parsedServiceConfig = createManagedChannelServiceConfig(serviceConfig); - interceptor.handleUpdate(parsedServiceConfig); - assertThat(interceptor.managedChannelServiceConfig.get().getDefaultMethodConfig()) - .isEqualTo(new MethodInfo(methodConfig, false, 1, 1)); - assertThat(interceptor.managedChannelServiceConfig.get().getServiceMap()).isEmpty(); - assertThat(interceptor.managedChannelServiceConfig.get().getServiceMethodMap()).isEmpty(); - } - - @Test - public void handleUpdate_replaceExistingConfig() { - JsonObj name1 = new JsonObj("service", "service"); - JsonObj methodConfig1 = new JsonObj("name", new JsonList(name1)); - JsonObj serviceConfig1 = new JsonObj("methodConfig", new JsonList(methodConfig1)); - - JsonObj name2 = new JsonObj("service", "service", "method", "method"); - JsonObj methodConfig2 = new JsonObj("name", new JsonList(name2)); - JsonObj serviceConfig2 = new JsonObj("methodConfig", new JsonList(methodConfig2)); - ManagedChannelServiceConfig parsedServiceConfig1 = - createManagedChannelServiceConfig(serviceConfig1); - ManagedChannelServiceConfig parsedServiceConfig2 = - createManagedChannelServiceConfig(serviceConfig2); - - interceptor.handleUpdate(parsedServiceConfig1); - - assertThat(interceptor.managedChannelServiceConfig.get().getServiceMap()).isNotEmpty(); - assertThat(interceptor.managedChannelServiceConfig.get().getServiceMethodMap()).isEmpty(); - - interceptor.handleUpdate(parsedServiceConfig2); - - assertThat(interceptor.managedChannelServiceConfig.get().getServiceMap()).isEmpty(); - assertThat(interceptor.managedChannelServiceConfig.get().getServiceMethodMap()).isNotEmpty(); - } - - @Test - public void handleUpdate_matchNames() { - JsonObj name1 = new JsonObj("service", "service2"); - JsonObj name2 = new JsonObj("service", "service", "method", "method"); - JsonObj methodConfig = new JsonObj("name", new JsonList(name1, name2)); - JsonObj serviceConfig = new JsonObj("methodConfig", new JsonList(methodConfig)); - ManagedChannelServiceConfig parsedServiceConfig = - createManagedChannelServiceConfig(serviceConfig); - - interceptor.handleUpdate(parsedServiceConfig); - - assertThat(interceptor.managedChannelServiceConfig.get().getServiceMethodMap()) - .containsExactly( - methodDescriptor.getFullMethodName(), - new MethodInfo(methodConfig, false, 1, 1)); - assertThat(interceptor.managedChannelServiceConfig.get().getServiceMap()).containsExactly( - "service2", new MethodInfo(methodConfig, false, 1, 1)); - } - - @Test - public void interceptCall_matchNames() { - JsonObj name0 = new JsonObj(); - JsonObj name1 = new JsonObj("service", "service"); - JsonObj name2 = new JsonObj("service", "service", "method", "method"); - JsonObj methodConfig0 = new JsonObj( - "name", new JsonList(name0), "maxRequestMessageBytes", 5d); - JsonObj methodConfig1 = new JsonObj( - "name", new JsonList(name1), "maxRequestMessageBytes", 6d); - JsonObj methodConfig2 = new JsonObj( - "name", new JsonList(name2), "maxRequestMessageBytes", 7d); - JsonObj serviceConfig = - new JsonObj("methodConfig", new JsonList(methodConfig0, methodConfig1, methodConfig2)); - ManagedChannelServiceConfig parsedServiceConfig = - createManagedChannelServiceConfig(serviceConfig); - - interceptor.handleUpdate(parsedServiceConfig); - - String fullMethodName = - MethodDescriptor.generateFullMethodName("service", "method"); - MethodDescriptor methodDescriptor = - MethodDescriptor.newBuilder(new NoopMarshaller(), new NoopMarshaller()) - .setType(MethodType.UNARY) - .setFullMethodName(fullMethodName) - .build(); - interceptor.interceptCall( - methodDescriptor, CallOptions.DEFAULT, channel); - verify(channel).newCall(eq(methodDescriptor), callOptionsCap.capture()); - assertThat(callOptionsCap.getValue().getMaxOutboundMessageSize()).isEqualTo(7); - - fullMethodName = - MethodDescriptor.generateFullMethodName("service", "method2"); - methodDescriptor = - MethodDescriptor.newBuilder(new NoopMarshaller(), new NoopMarshaller()) - .setType(MethodType.UNARY) - .setFullMethodName(fullMethodName) - .build(); - interceptor.interceptCall( - methodDescriptor, CallOptions.DEFAULT, channel); - verify(channel).newCall(eq(methodDescriptor), callOptionsCap.capture()); - assertThat(callOptionsCap.getValue().getMaxOutboundMessageSize()).isEqualTo(6); - - fullMethodName = - MethodDescriptor.generateFullMethodName("service2", "method"); - methodDescriptor = - MethodDescriptor.newBuilder(new NoopMarshaller(), new NoopMarshaller()) - .setType(MethodType.UNARY) - .setFullMethodName(fullMethodName) - .build(); - interceptor.interceptCall( - methodDescriptor, CallOptions.DEFAULT, channel); - verify(channel).newCall(eq(methodDescriptor), callOptionsCap.capture()); - assertThat(callOptionsCap.getValue().getMaxOutboundMessageSize()).isEqualTo(5); - } - - @Test - public void methodInfo_validateDeadline() { - JsonObj name = new JsonObj("service", "service"); - JsonObj methodConfig = new JsonObj("name", new JsonList(name), "timeout", "10000000000000000s"); - - thrown.expectMessage("Duration value is out of range"); - - new MethodInfo(methodConfig, false, 1, 1); - } - - @Test - public void methodInfo_saturateDeadline() { - JsonObj name = new JsonObj("service", "service"); - JsonObj methodConfig = new JsonObj("name", new JsonList(name), "timeout", "315576000000s"); - - MethodInfo info = new MethodInfo(methodConfig, false, 1, 1); - - assertThat(info.timeoutNanos).isEqualTo(Long.MAX_VALUE); - } - - @Test - public void methodInfo_badMaxRequestSize() { - JsonObj name = new JsonObj("service", "service"); - JsonObj methodConfig = new JsonObj("name", new JsonList(name), "maxRequestMessageBytes", -1d); - - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("exceeds bounds"); - - new MethodInfo(methodConfig, false, 1, 1); - } - - @Test - public void methodInfo_badMaxResponseSize() { - JsonObj name = new JsonObj("service", "service"); - JsonObj methodConfig = new JsonObj("name", new JsonList(name), "maxResponseMessageBytes", -1d); - - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("exceeds bounds"); - - new MethodInfo(methodConfig, false, 1, 1); - } - - private static ManagedChannelServiceConfig createManagedChannelServiceConfig( - JsonObj rawServiceConfig) { - // current tests doesn't use any other values except rawServiceConfig, so provide dummy values. - return ManagedChannelServiceConfig.fromServiceConfig( - rawServiceConfig, - /* retryEnabled= */ true, - /* maxRetryAttemptsLimit= */ 3, - /* maxHedgedAttemptsLimit= */ 4, - /* loadBalancingConfig= */ null); - } - - private static final class NoopMarshaller implements MethodDescriptor.Marshaller { - - @Override - public InputStream stream(Void value) { - return null; - } - - @Override - public Void parse(InputStream stream) { - return null; - } - } -} From 49e47a40897126f65c6313f19cd6b4388d531a4c Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Tue, 8 Sep 2020 18:24:20 -0700 Subject: [PATCH 33/86] Update README etc to reference 1.32.1 --- README.md | 28 ++++++++++++------------ cronet/README.md | 2 +- documentation/android-channel-builder.md | 4 ++-- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 164145d8c01..b1a66311d75 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,8 @@ For a guided tour, take a look at the [quick start guide](https://grpc.io/docs/languages/java/quickstart) or the more explanatory [gRPC basics](https://grpc.io/docs/languages/java/basics). -The [examples](https://github.com/grpc/grpc-java/tree/v1.31.1/examples) and the -[Android example](https://github.com/grpc/grpc-java/tree/v1.31.1/examples/android) +The [examples](https://github.com/grpc/grpc-java/tree/v1.32.1/examples) and the +[Android example](https://github.com/grpc/grpc-java/tree/v1.32.1/examples/android) are standalone projects that showcase the usage of gRPC. Download @@ -42,17 +42,17 @@ Download [the JARs][]. Or for Maven with non-Android, add to your `pom.xml`: io.grpc grpc-netty-shaded - 1.31.1 + 1.32.1 io.grpc grpc-protobuf - 1.31.1 + 1.32.1 io.grpc grpc-stub - 1.31.1 + 1.32.1 org.apache.tomcat @@ -64,23 +64,23 @@ Download [the JARs][]. Or for Maven with non-Android, add to your `pom.xml`: Or for Gradle with non-Android, add to your dependencies: ```gradle -implementation 'io.grpc:grpc-netty-shaded:1.31.1' -implementation 'io.grpc:grpc-protobuf:1.31.1' -implementation 'io.grpc:grpc-stub:1.31.1' +implementation 'io.grpc:grpc-netty-shaded:1.32.1' +implementation 'io.grpc:grpc-protobuf:1.32.1' +implementation 'io.grpc:grpc-stub:1.32.1' compileOnly 'org.apache.tomcat:annotations-api:6.0.53' // necessary for Java 9+ ``` For Android client, use `grpc-okhttp` instead of `grpc-netty-shaded` and `grpc-protobuf-lite` instead of `grpc-protobuf`: ```gradle -implementation 'io.grpc:grpc-okhttp:1.31.1' -implementation 'io.grpc:grpc-protobuf-lite:1.31.1' -implementation 'io.grpc:grpc-stub:1.31.1' +implementation 'io.grpc:grpc-okhttp:1.32.1' +implementation 'io.grpc:grpc-protobuf-lite:1.32.1' +implementation 'io.grpc:grpc-stub:1.32.1' compileOnly 'org.apache.tomcat:annotations-api:6.0.53' // necessary for Java 9+ ``` [the JARs]: -https://search.maven.org/search?q=g:io.grpc%20AND%20v:1.31.1 +https://search.maven.org/search?q=g:io.grpc%20AND%20v:1.32.1 Development snapshots are available in [Sonatypes's snapshot repository](https://oss.sonatype.org/content/repositories/snapshots/). @@ -112,7 +112,7 @@ For protobuf-based codegen integrated with the Maven build system, you can use com.google.protobuf:protoc:3.12.0:exe:${os.detected.classifier} grpc-java - io.grpc:protoc-gen-grpc-java:1.31.1:exe:${os.detected.classifier} + io.grpc:protoc-gen-grpc-java:1.32.1:exe:${os.detected.classifier} @@ -142,7 +142,7 @@ protobuf { } plugins { grpc { - artifact = 'io.grpc:protoc-gen-grpc-java:1.31.1' + artifact = 'io.grpc:protoc-gen-grpc-java:1.32.1' } } generateProtoTasks { diff --git a/cronet/README.md b/cronet/README.md index 5cfe8858bd6..b752082c42f 100644 --- a/cronet/README.md +++ b/cronet/README.md @@ -26,7 +26,7 @@ In your app module's `build.gradle` file, include a dependency on both `grpc-cro Google Play Services Client Library for Cronet ``` -implementation 'io.grpc:grpc-cronet:1.31.1' +implementation 'io.grpc:grpc-cronet:1.32.1' implementation 'com.google.android.gms:play-services-cronet:16.0.0' ``` diff --git a/documentation/android-channel-builder.md b/documentation/android-channel-builder.md index 9e6e233806f..579f4f13210 100644 --- a/documentation/android-channel-builder.md +++ b/documentation/android-channel-builder.md @@ -36,8 +36,8 @@ In your `build.gradle` file, include a dependency on both `grpc-android` and `grpc-okhttp`: ``` -implementation 'io.grpc:grpc-android:1.31.1' -implementation 'io.grpc:grpc-okhttp:1.31.1' +implementation 'io.grpc:grpc-android:1.32.1' +implementation 'io.grpc:grpc-okhttp:1.32.1' ``` You also need permission to access the device's network state in your From 73dd3672fc0caf0885c7dbd26ab4ee47291d929c Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Thu, 10 Sep 2020 15:11:19 -0700 Subject: [PATCH 34/86] core: fix drainPendingCalls might be called twice Fixing the bug: if two consecutive name resolution updates are queued together in SynchronizationContext, drainPendingCalls() might be called twice and be broken. --- core/src/main/java/io/grpc/internal/ManagedChannelImpl.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java b/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java index b3a69bbce9b..10aa197efb6 100644 --- a/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java +++ b/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java @@ -1568,6 +1568,7 @@ public void run() { Status serviceConfigError = configOrError != null ? configOrError.getError() : null; ManagedChannelServiceConfig effectiveServiceConfig; + InternalConfigSelector prevConfigSelector = configSelector.get(); if (!lookUpServiceConfig) { if (validServiceConfig != null) { channelLogger.log( @@ -1639,7 +1640,9 @@ public void run() { re); } } - realChannel.drainPendingCalls(); + if (prevConfigSelector == INITIAL_PENDING_SELECTOR) { + realChannel.drainPendingCalls(); + } Attributes effectiveAttrs = resolutionResult.getAttributes(); // Call LB only if it's not shutdown. If LB is shutdown, lbHelper won't match. From 234125ee39b8d35058e9fee6c6aa9f12923b6f05 Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Fri, 11 Sep 2020 09:35:18 -0700 Subject: [PATCH 35/86] core: re-organize RealChannel with updateConfigSelector() As mentioned in https://github.com/grpc/grpc-java/pull/7413#issuecomment-690756200 `RealChannel` did not manage `configSelector`, and therefore `configSelector.get()`, `configSelector.set()` and `drainPendingCalls()` were scattered everywhere in `ManagedChannelImpl`. This PR re-organizes `RealChannel` to manage `configSelector`. --- .../io/grpc/internal/ManagedChannelImpl.java | 47 ++++++++++--------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java b/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java index 10aa197efb6..494899b48c0 100644 --- a/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java +++ b/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java @@ -263,10 +263,6 @@ public void uncaughtException(Thread t, Throwable e) { // Must be mutated and read from constructor or syncContext // used for channel tracing when value changed private ManagedChannelServiceConfig lastServiceConfig = EMPTY_SERVICE_CONFIG; - // Reference to null if no config selector is available from resolution result - // Reference must be set() from syncContext - private final AtomicReference configSelector = - new AtomicReference<>(INITIAL_PENDING_SELECTOR); @Nullable private final ManagedChannelServiceConfig defaultServiceConfig; @@ -750,7 +746,7 @@ static NameResolver getNameResolver(String target, NameResolver.Factory nameReso @VisibleForTesting InternalConfigSelector getConfigSelector() { - return configSelector.get(); + return realChannel.configSelector.get(); } /** @@ -891,6 +887,10 @@ private Executor getCallExecutor(CallOptions callOptions) { } private class RealChannel extends Channel { + // Reference to null if no config selector is available from resolution result + // Reference must be set() from syncContext + private final AtomicReference configSelector = + new AtomicReference<>(INITIAL_PENDING_SELECTOR); // Set when the NameResolver is initially created. When we create a new NameResolver for the // same target, the new instance must have the same value. private final String authority; @@ -954,12 +954,20 @@ public void run() { } // Must run in SynchronizationContext. - private void drainPendingCalls() { - if (pendingCalls == null) { - return; + void updateConfigSelector(@Nullable InternalConfigSelector config) { + InternalConfigSelector prevConfig = configSelector.get(); + configSelector.set(config); + if (prevConfig == INITIAL_PENDING_SELECTOR && pendingCalls != null) { + for (RealChannel.PendingCall pendingCall : pendingCalls) { + pendingCall.reprocess(); + } } - for (RealChannel.PendingCall pendingCall : pendingCalls) { - pendingCall.reprocess(); + } + + // Must run in SynchronizationContext. + void onConfigError() { + if (configSelector.get() == INITIAL_PENDING_SELECTOR) { + updateConfigSelector(null); } } @@ -1568,7 +1576,6 @@ public void run() { Status serviceConfigError = configOrError != null ? configOrError.getError() : null; ManagedChannelServiceConfig effectiveServiceConfig; - InternalConfigSelector prevConfigSelector = configSelector.get(); if (!lookUpServiceConfig) { if (validServiceConfig != null) { channelLogger.log( @@ -1582,14 +1589,14 @@ public void run() { ChannelLogLevel.INFO, "Config selector from name resolver discarded by channel settings"); } - configSelector.set(effectiveServiceConfig.getDefaultConfigSelector()); + realChannel.updateConfigSelector(effectiveServiceConfig.getDefaultConfigSelector()); } else { // Try to use config if returned from name resolver // Otherwise, try to use the default config if available if (validServiceConfig != null) { effectiveServiceConfig = validServiceConfig; if (resolvedConfigSelector != null) { - configSelector.set(resolvedConfigSelector); + realChannel.updateConfigSelector(resolvedConfigSelector); if (effectiveServiceConfig.getDefaultConfigSelector() != null) { channelLogger.log( ChannelLogLevel.DEBUG, @@ -1597,11 +1604,11 @@ public void run() { + "config-selector"); } } else { - configSelector.set(effectiveServiceConfig.getDefaultConfigSelector()); + realChannel.updateConfigSelector(effectiveServiceConfig.getDefaultConfigSelector()); } } else if (defaultServiceConfig != null) { effectiveServiceConfig = defaultServiceConfig; - configSelector.set(effectiveServiceConfig.getDefaultConfigSelector()); + realChannel.updateConfigSelector(effectiveServiceConfig.getDefaultConfigSelector()); channelLogger.log( ChannelLogLevel.INFO, "Received no service config, using default service config"); @@ -1618,7 +1625,7 @@ public void run() { } } else { effectiveServiceConfig = EMPTY_SERVICE_CONFIG; - configSelector.set(null); + realChannel.updateConfigSelector(null); } if (!effectiveServiceConfig.equals(lastServiceConfig)) { channelLogger.log( @@ -1640,9 +1647,6 @@ public void run() { re); } } - if (prevConfigSelector == INITIAL_PENDING_SELECTOR) { - realChannel.drainPendingCalls(); - } Attributes effectiveAttrs = resolutionResult.getAttributes(); // Call LB only if it's not shutdown. If LB is shutdown, lbHelper won't match. @@ -1690,10 +1694,7 @@ public void run() { private void handleErrorInSyncContext(Status error) { logger.log(Level.WARNING, "[{0}] Failed to resolve name. status={1}", new Object[] {getLogId(), error}); - if (configSelector.get() == INITIAL_PENDING_SELECTOR) { - configSelector.set(null); - realChannel.drainPendingCalls(); - } + realChannel.onConfigError(); if (lastResolutionState != ResolutionState.ERROR) { channelLogger.log(ChannelLogLevel.WARNING, "Failed to resolve name: {0}", error); lastResolutionState = ResolutionState.ERROR; From 8ac63626c3a9221f142d58317d94835999ee1bae Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Fri, 11 Sep 2020 10:08:52 -0700 Subject: [PATCH 36/86] xds: make channel creds required in bootstrap file (#7396) --- .../main/java/io/grpc/xds/Bootstrapper.java | 25 ++++++++++--------- .../java/io/grpc/xds/BootstrapperTest.java | 19 +++++++++++--- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/Bootstrapper.java b/xds/src/main/java/io/grpc/xds/Bootstrapper.java index a0bacd4190b..ea59498a272 100644 --- a/xds/src/main/java/io/grpc/xds/Bootstrapper.java +++ b/xds/src/main/java/io/grpc/xds/Bootstrapper.java @@ -101,19 +101,20 @@ public static BootstrapInfo parseConfig(String rawData) throws IOException { logger.log(XdsLogLevel.INFO, "xDS server URI: {0}", serverUri); List channelCredsOptions = new ArrayList<>(); List rawChannelCredsList = JsonUtil.getList(serverConfig, "channel_creds"); - // List of channel creds is optional. - if (rawChannelCredsList != null) { - List> channelCredsList = JsonUtil.checkObjectList(rawChannelCredsList); - for (Map channelCreds : channelCredsList) { - String type = JsonUtil.getString(channelCreds, "type"); - if (type == null) { - throw new IOException("Invalid bootstrap: 'xds_servers' contains server with " - + "unknown type 'channel_creds'."); - } - logger.log(XdsLogLevel.INFO, "Channel credentials option: {0}", type); - ChannelCreds creds = new ChannelCreds(type, JsonUtil.getObject(channelCreds, "config")); - channelCredsOptions.add(creds); + if (rawChannelCredsList == null || rawChannelCredsList.isEmpty()) { + throw new IOException( + "Invalid bootstrap: server " + serverUri + " 'channel_creds' required"); + } + List> channelCredsList = JsonUtil.checkObjectList(rawChannelCredsList); + for (Map channelCreds : channelCredsList) { + String type = JsonUtil.getString(channelCreds, "type"); + if (type == null) { + throw new IOException( + "Invalid bootstrap: server " + serverUri + " with 'channel_creds' type unspecified"); } + logger.log(XdsLogLevel.INFO, "Channel credentials option: {0}", type); + ChannelCreds creds = new ChannelCreds(type, JsonUtil.getObject(channelCreds, "config")); + channelCredsOptions.add(creds); } List serverFeatures = JsonUtil.getListOfStrings(serverConfig, "server_features"); if (serverFeatures != null) { diff --git a/xds/src/test/java/io/grpc/xds/BootstrapperTest.java b/xds/src/test/java/io/grpc/xds/BootstrapperTest.java index 17f42aad73b..75b250f4f04 100644 --- a/xds/src/test/java/io/grpc/xds/BootstrapperTest.java +++ b/xds/src/test/java/io/grpc/xds/BootstrapperTest.java @@ -24,6 +24,7 @@ import io.grpc.internal.GrpcUtil; import io.grpc.internal.GrpcUtil.GrpcBuildVersion; import io.grpc.xds.Bootstrapper.BootstrapInfo; +import io.grpc.xds.Bootstrapper.ChannelCreds; import io.grpc.xds.Bootstrapper.ServerInfo; import io.grpc.xds.EnvoyProtoData.Locality; import io.grpc.xds.EnvoyProtoData.Node; @@ -121,7 +122,9 @@ public void parseBootstrap_validData_multipleXdsServers() throws IOException { + " },\n" + " {\n" + " \"server_uri\": \"trafficdirector-bar.googleapis.com:443\",\n" - + " \"channel_creds\": []\n" + + " \"channel_creds\": [\n" + + " {\"type\": \"insecure\"}" + + " ]\n" + " }\n" + " ]\n" + "}"; @@ -142,7 +145,9 @@ public void parseBootstrap_validData_multipleXdsServers() throws IOException { assertThat(serverInfoList.get(0).getServerFeatures()).contains("xds_v3"); assertThat(serverInfoList.get(1).getServerUri()) .isEqualTo("trafficdirector-bar.googleapis.com:443"); - assertThat(serverInfoList.get(1).getChannelCredentials()).isEmpty(); + assertThat(serverInfoList.get(1).getChannelCredentials().get(0).getType()) + .isEqualTo("insecure"); + assertThat(serverInfoList.get(0).getChannelCredentials().get(0).getConfig()).isNull(); assertThat(info.getNode()).isEqualTo( getNodeBuilder() .setId("ENVOY_NODE_ID") @@ -234,7 +239,10 @@ public void parseBootstrap_minimalUsableData() throws IOException { String rawData = "{\n" + " \"xds_servers\": [\n" + " {\n" - + " \"server_uri\": \"trafficdirector.googleapis.com:443\"\n" + + " \"server_uri\": \"trafficdirector.googleapis.com:443\",\n" + + " \"channel_creds\": [\n" + + " {\"type\": \"insecure\"}\n" + + " ]\n" + " }\n" + " ]\n" + "}"; @@ -243,7 +251,10 @@ public void parseBootstrap_minimalUsableData() throws IOException { assertThat(info.getServers()).hasSize(1); ServerInfo serverInfo = Iterables.getOnlyElement(info.getServers()); assertThat(serverInfo.getServerUri()).isEqualTo("trafficdirector.googleapis.com:443"); - assertThat(serverInfo.getChannelCredentials()).isEmpty(); + assertThat(serverInfo.getChannelCredentials()).hasSize(1); + ChannelCreds creds = Iterables.getOnlyElement(serverInfo.getChannelCredentials()); + assertThat(creds.getType()).isEqualTo("insecure"); + assertThat(creds.getConfig()).isNull(); assertThat(info.getNode()).isEqualTo(getNodeBuilder().build()); } From b927278f22370830260880ed74d8064fa224e841 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Fri, 11 Sep 2020 16:21:09 -0400 Subject: [PATCH 37/86] core: cleanup AbstractManagedChannelImplBuilderTest --- ...AbstractManagedChannelImplBuilderTest.java | 22 ++----------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/core/src/test/java/io/grpc/internal/AbstractManagedChannelImplBuilderTest.java b/core/src/test/java/io/grpc/internal/AbstractManagedChannelImplBuilderTest.java index ba9ac5f4a7a..2ac49c48ce3 100644 --- a/core/src/test/java/io/grpc/internal/AbstractManagedChannelImplBuilderTest.java +++ b/core/src/test/java/io/grpc/internal/AbstractManagedChannelImplBuilderTest.java @@ -67,8 +67,8 @@ public ClientCall interceptCall( } }; - private Builder builder = new Builder("fake"); - private Builder directAddressBuilder = new Builder(new SocketAddress(){}, "fake"); + private final Builder builder = new Builder("fake"); + private final Builder directAddressBuilder = new Builder(new SocketAddress(){}, "fake"); @Test public void executor_default() { @@ -261,7 +261,6 @@ public void overrideAuthority_invalid() { @Test public void overrideAuthority_getNameResolverFactory() { - Builder builder = new Builder("target"); assertNull(builder.authorityOverride); assertFalse(builder.getNameResolverFactory() instanceof OverrideAuthorityNameResolverFactory); builder.overrideAuthority("google.com"); @@ -323,8 +322,6 @@ public void getEffectiveInterceptors_disableBoth() { @Test public void idleTimeout() { - Builder builder = new Builder("target"); - assertEquals(AbstractManagedChannelImplBuilder.IDLE_MODE_DEFAULT_TIMEOUT_MILLIS, builder.getIdleTimeoutMillis()); @@ -352,7 +349,6 @@ public void idleTimeout() { @Test public void maxRetryAttempts() { - Builder builder = new Builder("target"); assertEquals(5, builder.maxRetryAttempts); builder.maxRetryAttempts(3); @@ -361,7 +357,6 @@ public void maxRetryAttempts() { @Test public void maxHedgedAttempts() { - Builder builder = new Builder("target"); assertEquals(5, builder.maxHedgedAttempts); builder.maxHedgedAttempts(3); @@ -370,7 +365,6 @@ public void maxHedgedAttempts() { @Test public void retryBufferSize() { - Builder builder = new Builder("target"); assertEquals(1L << 24, builder.retryBufferSize); builder.retryBufferSize(3456L); @@ -379,7 +373,6 @@ public void retryBufferSize() { @Test public void perRpcBufferLimit() { - Builder builder = new Builder("target"); assertEquals(1L << 20, builder.perRpcBufferLimit); builder.perRpcBufferLimit(3456L); @@ -388,24 +381,18 @@ public void perRpcBufferLimit() { @Test public void retryBufferSizeInvalidArg() { - Builder builder = new Builder("target"); - thrown.expect(IllegalArgumentException.class); builder.retryBufferSize(0L); } @Test public void perRpcBufferLimitInvalidArg() { - Builder builder = new Builder("target"); - thrown.expect(IllegalArgumentException.class); builder.perRpcBufferLimit(0L); } @Test public void disableRetry() { - Builder builder = new Builder("target"); - builder.enableRetry(); assertTrue(builder.retryEnabled); @@ -421,7 +408,6 @@ public void disableRetry() { @Test public void defaultServiceConfig_nullKey() { - Builder builder = new Builder("target"); Map config = new HashMap<>(); config.put(null, "val"); @@ -431,7 +417,6 @@ public void defaultServiceConfig_nullKey() { @Test public void defaultServiceConfig_intKey() { - Builder builder = new Builder("target"); Map subConfig = new HashMap<>(); subConfig.put(3, "val"); Map config = new HashMap<>(); @@ -443,7 +428,6 @@ public void defaultServiceConfig_intKey() { @Test public void defaultServiceConfig_intValue() { - Builder builder = new Builder("target"); Map config = new HashMap<>(); config.put("key", 3); @@ -453,7 +437,6 @@ public void defaultServiceConfig_intValue() { @Test public void defaultServiceConfig_nested() { - Builder builder = new Builder("target"); Map config = new HashMap<>(); List list1 = new ArrayList<>(); list1.add(123D); @@ -476,7 +459,6 @@ public void defaultServiceConfig_nested() { @Test public void disableNameResolverServiceConfig() { - Builder builder = new Builder("target"); assertThat(builder.lookUpServiceConfig).isTrue(); builder.disableServiceConfigLookUp(); From 69e8204066fd4bfc442b7abda80a0802ea21ae61 Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Fri, 11 Sep 2020 14:56:03 -0700 Subject: [PATCH 38/86] xds: promote XdsNameResolver2 (#7416) --- .../grpc/xds/ClusterManagerLoadBalancer.java | 2 +- .../java/io/grpc/xds/XdsNameResolver.java | 496 +++++++------ .../java/io/grpc/xds/XdsNameResolver2.java | 374 ---------- .../io/grpc/xds/XdsNameResolverProvider.java | 65 +- .../io/grpc/xds/XdsNameResolverProvider2.java | 132 ---- .../xds/ClusterManagerLoadBalancerTest.java | 2 +- .../io/grpc/xds/XdsNameResolver2Test.java | 554 -------------- .../xds/XdsNameResolverIntegrationTest.java | 570 -------------- .../java/io/grpc/xds/XdsNameResolverTest.java | 698 ++++++++++++------ 9 files changed, 804 insertions(+), 2089 deletions(-) delete mode 100644 xds/src/main/java/io/grpc/xds/XdsNameResolver2.java delete mode 100644 xds/src/main/java/io/grpc/xds/XdsNameResolverProvider2.java delete mode 100644 xds/src/test/java/io/grpc/xds/XdsNameResolver2Test.java delete mode 100644 xds/src/test/java/io/grpc/xds/XdsNameResolverIntegrationTest.java diff --git a/xds/src/main/java/io/grpc/xds/ClusterManagerLoadBalancer.java b/xds/src/main/java/io/grpc/xds/ClusterManagerLoadBalancer.java index f8901fb0ebe..91b7a93151f 100644 --- a/xds/src/main/java/io/grpc/xds/ClusterManagerLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/ClusterManagerLoadBalancer.java @@ -145,7 +145,7 @@ private void updateOverallBalancingState() { @Override public PickResult pickSubchannel(PickSubchannelArgs args) { String clusterName = - args.getCallOptions().getOption(XdsNameResolver2.CLUSTER_SELECTION_KEY); + args.getCallOptions().getOption(XdsNameResolver.CLUSTER_SELECTION_KEY); SubchannelPicker delegate = childPickers.get(clusterName); if (delegate == null) { return diff --git a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java index a94360848fe..22f74f7f795 100644 --- a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java +++ b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java @@ -19,44 +19,38 @@ import static com.google.common.base.Preconditions.checkNotNull; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Stopwatch; -import com.google.common.base.Supplier; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; import com.google.gson.Gson; -import com.google.re2j.Pattern; import io.grpc.Attributes; -import io.grpc.EquivalentAddressGroup; +import io.grpc.CallOptions; +import io.grpc.InternalConfigSelector; import io.grpc.InternalLogId; +import io.grpc.LoadBalancer.PickSubchannelArgs; +import io.grpc.Metadata; import io.grpc.NameResolver; import io.grpc.Status; import io.grpc.SynchronizationContext; -import io.grpc.internal.BackoffPolicy; import io.grpc.internal.GrpcUtil; import io.grpc.internal.ObjectPool; import io.grpc.xds.Bootstrapper.BootstrapInfo; -import io.grpc.xds.Bootstrapper.ServerInfo; import io.grpc.xds.EnvoyProtoData.ClusterWeight; -import io.grpc.xds.EnvoyProtoData.Node; import io.grpc.xds.EnvoyProtoData.Route; import io.grpc.xds.EnvoyProtoData.RouteAction; -import io.grpc.xds.RouteMatch.FractionMatcher; -import io.grpc.xds.RouteMatch.HeaderMatcher; -import io.grpc.xds.RouteMatch.PathMatcher; +import io.grpc.xds.ThreadSafeRandom.ThreadSafeRandomImpl; import io.grpc.xds.XdsClient.ConfigUpdate; import io.grpc.xds.XdsClient.ConfigWatcher; -import io.grpc.xds.XdsClient.RefCountedXdsClientObjectPool; -import io.grpc.xds.XdsClient.XdsChannelFactory; -import io.grpc.xds.XdsClient.XdsClientFactory; +import io.grpc.xds.XdsClient.XdsClientPoolFactory; import io.grpc.xds.XdsLogger.XdsLogLevel; -import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.LinkedHashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.concurrent.ScheduledExecutorService; -import javax.annotation.Nullable; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; /** * A {@link NameResolver} for resolving gRPC target names with "xds:" scheme. @@ -68,36 +62,45 @@ */ final class XdsNameResolver extends NameResolver { + static final CallOptions.Key CLUSTER_SELECTION_KEY = + CallOptions.Key.create("io.grpc.xds.CLUSTER_SELECTION_KEY"); + private final XdsLogger logger; private final String authority; - private final XdsChannelFactory channelFactory; - private final SynchronizationContext syncContext; - private final ScheduledExecutorService timeService; private final ServiceConfigParser serviceConfigParser; - private final BackoffPolicy.Provider backoffPolicyProvider; - private final Supplier stopwatchSupplier; + private final SynchronizationContext syncContext; private final Bootstrapper bootstrapper; + private final XdsClientPoolFactory xdsClientPoolFactory; + private final ThreadSafeRandom random; + private final ConcurrentMap clusterRefs = new ConcurrentHashMap<>(); + private final ConfigSelector configSelector = new ConfigSelector(); - @Nullable + private volatile List routes = Collections.emptyList(); + private Listener2 listener; private ObjectPool xdsClientPool; - @Nullable private XdsClient xdsClient; + XdsNameResolver(String name, + ServiceConfigParser serviceConfigParser, + SynchronizationContext syncContext, + XdsClientPoolFactory xdsClientPoolFactory) { + this(name, serviceConfigParser, syncContext, Bootstrapper.getInstance(), + xdsClientPoolFactory, ThreadSafeRandomImpl.instance); + } + XdsNameResolver( String name, - Args args, - BackoffPolicy.Provider backoffPolicyProvider, - Supplier stopwatchSupplier, - XdsChannelFactory channelFactory, - Bootstrapper bootstrapper) { + ServiceConfigParser serviceConfigParser, + SynchronizationContext syncContext, + Bootstrapper bootstrapper, + XdsClientPoolFactory xdsClientPoolFactory, + ThreadSafeRandom random) { authority = GrpcUtil.checkAuthority(checkNotNull(name, "name")); - this.channelFactory = checkNotNull(channelFactory, "channelFactory"); - this.syncContext = checkNotNull(args.getSynchronizationContext(), "syncContext"); - this.timeService = checkNotNull(args.getScheduledExecutorService(), "timeService"); - this.serviceConfigParser = checkNotNull(args.getServiceConfigParser(), "serviceConfigParser"); - this.backoffPolicyProvider = checkNotNull(backoffPolicyProvider, "backoffPolicyProvider"); - this.stopwatchSupplier = checkNotNull(stopwatchSupplier, "stopwatchSupplier"); + this.serviceConfigParser = checkNotNull(serviceConfigParser, "serviceConfigParser"); + this.syncContext = checkNotNull(syncContext, "syncContext"); this.bootstrapper = checkNotNull(bootstrapper, "bootstrapper"); + this.xdsClientPoolFactory = checkNotNull(xdsClientPoolFactory, "xdsClientPoolFactory"); + this.random = checkNotNull(random, "random"); logger = XdsLogger.withLogId(InternalLogId.allocate("xds-resolver", name)); logger.log(XdsLogLevel.INFO, "Created resolver for {0}", name); } @@ -109,78 +112,242 @@ public String getServiceAuthority() { @Override public void start(Listener2 listener) { + this.listener = checkNotNull(listener, "listener"); BootstrapInfo bootstrapInfo; try { bootstrapInfo = bootstrapper.readBootstrap(); } catch (Exception e) { - listener.onError(Status.UNAVAILABLE.withDescription("Failed to bootstrap").withCause(e)); - return; - } - final List serverList = bootstrapInfo.getServers(); - final Node node = bootstrapInfo.getNode(); - if (serverList.isEmpty()) { listener.onError( - Status.UNAVAILABLE.withDescription("No management server provided by bootstrap")); + Status.UNAVAILABLE.withDescription("Failed to load xDS bootstrap").withCause(e)); return; } - - XdsClientFactory xdsClientFactory = new XdsClientFactory() { - @Override - XdsClient createXdsClient() { - return - new XdsClientImpl( - authority, - serverList, - channelFactory, - node, - syncContext, - timeService, - backoffPolicyProvider, - stopwatchSupplier); - } - }; - xdsClientPool = new RefCountedXdsClientObjectPool(xdsClientFactory); + xdsClientPool = xdsClientPoolFactory.newXdsClientObjectPool(bootstrapInfo); xdsClient = xdsClientPool.getObject(); - xdsClient.watchConfigData(authority, new ConfigWatcherImpl(listener)); + xdsClient.watchConfigData(authority, new ConfigWatcherImpl()); } - private class ConfigWatcherImpl implements ConfigWatcher { + @Override + public void shutdown() { + logger.log(XdsLogLevel.INFO, "Shutdown"); + if (xdsClient != null) { + xdsClient = xdsClientPool.returnObject(xdsClient); + } + } - final Listener2 listener; + @VisibleForTesting + static Map generateServiceConfigWithMethodTimeoutConfig(long timeoutNano) { + String timeout = timeoutNano / 1_000_000_000.0 + "s"; + Map methodConfig = new HashMap<>(); + methodConfig.put( + "name", Collections.singletonList(Collections.emptyMap())); + methodConfig.put("timeout", timeout); + return Collections.singletonMap( + "methodConfig", Collections.singletonList(Collections.unmodifiableMap(methodConfig))); + } - ConfigWatcherImpl(Listener2 listener) { - this.listener = listener; + @VisibleForTesting + static Map generateServiceConfigWithLoadBalancingConfig(Collection clusters) { + Map childPolicy = new HashMap<>(); + for (String cluster : clusters) { + List>> lbPolicy = + Collections.singletonList( + Collections.singletonMap( + "cds_experimental", Collections.singletonMap("cluster", cluster))); + childPolicy.put(cluster, Collections.singletonMap("lbPolicy", lbPolicy)); } + return Collections.singletonMap("loadBalancingConfig", + Collections.singletonList( + Collections.singletonMap( + "cluster_manager_experimental", Collections.singletonMap( + "childPolicy", Collections.unmodifiableMap(childPolicy))))); + } - @Override - public void onConfigChanged(ConfigUpdate update) { + @VisibleForTesting + XdsClient getXdsClient() { + return xdsClient; + } + + private void updateResolutionResult() { + Map rawServiceConfig = + generateServiceConfigWithLoadBalancingConfig(clusterRefs.keySet()); + if (logger.isLoggable(XdsLogLevel.INFO)) { logger.log( - XdsLogLevel.INFO, - "Received config update with {0} routes from xDS client {1}", - update.getRoutes().size(), - xdsClient); - Map rawLbConfig = generateXdsRoutingRawConfig(update.getRoutes()); - Map serviceConfig = - ImmutableMap.of("loadBalancingConfig", ImmutableList.of(rawLbConfig)); + XdsLogLevel.INFO, "Generated service config:\n{0}", new Gson().toJson(rawServiceConfig)); + } + ConfigOrError parsedServiceConfig = serviceConfigParser.parseServiceConfig(rawServiceConfig); + Attributes attrs = + Attributes.newBuilder() + .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) + .set(InternalConfigSelector.KEY, configSelector) + .build(); + ResolutionResult result = + ResolutionResult.newBuilder() + .setAttributes(attrs) + .setServiceConfig(parsedServiceConfig) + .build(); + listener.onResult(result); + } + + private final class ConfigSelector extends InternalConfigSelector { + @Override + public Result selectConfig(PickSubchannelArgs args) { + // Index ASCII headers by keys. + Map> asciiHeaders = new HashMap<>(); + Metadata headers = args.getHeaders(); + for (String headerName : headers.keys()) { + if (headerName.endsWith(Metadata.BINARY_HEADER_SUFFIX)) { + continue; + } + Metadata.Key key = Metadata.Key.of(headerName, Metadata.ASCII_STRING_MARSHALLER); + asciiHeaders.put(headerName, headers.getAll(key)); + } + String cluster = null; + Route selectedRoute = null; + do { + for (Route route : routes) { + if (route.getRouteMatch().matches( + "/" + args.getMethodDescriptor().getFullMethodName(), asciiHeaders)) { + selectedRoute = route; + break; + } + } + if (selectedRoute == null) { + return Result.forError( + Status.UNAVAILABLE.withDescription("Could not find xDS route matching RPC")); + } + RouteAction action = selectedRoute.getRouteAction(); + if (action.getCluster() != null) { + cluster = action.getCluster(); + } else if (action.getWeightedCluster() != null) { + int totalWeight = 0; + for (ClusterWeight weightedCluster : action.getWeightedCluster()) { + totalWeight += weightedCluster.getWeight(); + } + int select = random.nextInt(totalWeight); + int accumulator = 0; + for (ClusterWeight weightedCluster : action.getWeightedCluster()) { + accumulator += weightedCluster.getWeight(); + if (select < accumulator) { + cluster = weightedCluster.getName(); + break; + } + } + } + } while (!retainCluster(cluster)); + // TODO(chengyuanzhang): avoid service config generation and parsing for each call. + Map rawServiceConfig = + generateServiceConfigWithMethodTimeoutConfig( + selectedRoute.getRouteAction().getTimeoutNano()); if (logger.isLoggable(XdsLogLevel.INFO)) { - logger.log( - XdsLogLevel.INFO, - "Generated service config:\n{0}", - new Gson().toJson(serviceConfig)); + logger.log(XdsLogLevel.INFO, + "Generated service config (method config):\n{0}", new Gson().toJson(rawServiceConfig)); + } + ConfigOrError parsedServiceConfig = serviceConfigParser.parseServiceConfig(rawServiceConfig); + Object config = parsedServiceConfig.getConfig(); + if (config == null) { + releaseCluster(cluster); + return Result.forError( + parsedServiceConfig.getError().augmentDescription( + "Failed to parse service config (method config)")); + } + final String finalCluster = cluster; + class SelectionCompleted implements Runnable { + @Override + public void run() { + releaseCluster(finalCluster); + } } - Attributes attrs = - Attributes.newBuilder() - .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) - .build(); - ConfigOrError parsedServiceConfig = serviceConfigParser.parseServiceConfig(serviceConfig); - ResolutionResult result = - ResolutionResult.newBuilder() - .setAddresses(ImmutableList.of()) - .setAttributes(attrs) - .setServiceConfig(parsedServiceConfig) + return + Result.newBuilder() + .setCallOptions(args.getCallOptions().withOption(CLUSTER_SELECTION_KEY, cluster)) + .setConfig(config) + .setCommittedCallback(new SelectionCompleted()) .build(); - listener.onResult(result); + } + + private boolean retainCluster(String cluster) { + AtomicInteger refCount = clusterRefs.get(cluster); + if (refCount == null) { + return false; + } + int count; + do { + count = refCount.get(); + if (count == 0) { + return false; + } + } while (!refCount.compareAndSet(count, count + 1)); + return true; + } + + private void releaseCluster(final String cluster) { + int count = clusterRefs.get(cluster).decrementAndGet(); + if (count == 0) { + syncContext.execute(new Runnable() { + @Override + public void run() { + if (clusterRefs.get(cluster).get() == 0) { + clusterRefs.remove(cluster); + updateResolutionResult(); + } + } + }); + } + } + } + + // https://github.com/google/error-prone/issues/1767 + @SuppressWarnings("ModifyCollectionInEnhancedForLoop") + private class ConfigWatcherImpl implements ConfigWatcher { + private Set existingClusters; + + @Override + public void onConfigChanged(ConfigUpdate update) { + Set clusters = new HashSet<>(); + for (Route route : update.getRoutes()) { + RouteAction action = route.getRouteAction(); + if (action.getCluster() != null) { + clusters.add(action.getCluster()); + } else if (action.getWeightedCluster() != null) { + for (ClusterWeight weighedCluster : action.getWeightedCluster()) { + clusters.add(weighedCluster.getName()); + } + } + } + Set addedClusters = + existingClusters == null ? clusters : Sets.difference(clusters, existingClusters); + Set deletedClusters = + existingClusters == null + ? Collections.emptySet() : Sets.difference(existingClusters, clusters); + existingClusters = clusters; + boolean shouldUpdateResult = false; + for (String cluster : addedClusters) { + if (clusterRefs.containsKey(cluster)) { + clusterRefs.get(cluster).incrementAndGet(); + } else { + clusterRefs.put(cluster, new AtomicInteger(1)); + shouldUpdateResult = true; + } + } + // Update service config to include newly added clusters. + if (shouldUpdateResult) { + updateResolutionResult(); + } + // Make newly added clusters selectable by config selector and deleted clusters no longer + // selectable. + routes = update.getRoutes(); + shouldUpdateResult = false; + for (String cluster : deletedClusters) { + int count = clusterRefs.get(cluster).decrementAndGet(); + if (count == 0) { + clusterRefs.remove(cluster); + shouldUpdateResult = true; + } + } + if (shouldUpdateResult) { + updateResolutionResult(); + } } @Override @@ -191,6 +358,7 @@ public void onResourceDoesNotExist(String resourceName) { ResolutionResult result = ResolutionResult.newBuilder() .setServiceConfig(parsedServiceConfig) + // let channel take action for no config selector .build(); listener.onResult(result); } @@ -203,152 +371,4 @@ public void onError(Status error) { listener.onError(error); } } - - @VisibleForTesting - static ImmutableMap generateXdsRoutingRawConfig(List routes) { - List rawRoutes = new ArrayList<>(); - Map rawActions = new LinkedHashMap<>(); - Map existingActions = new HashMap<>(); - for (Route route : routes) { - RouteAction routeAction = route.getRouteAction(); - String actionName; - Map actionPolicy; - if (existingActions.containsKey(routeAction)) { - actionName = existingActions.get(routeAction); - } else { - if (routeAction.getCluster() != null) { - actionName = "cds:" + routeAction.getCluster(); - actionPolicy = generateCdsRawConfig(routeAction.getCluster()); - } else { - StringBuilder sb = new StringBuilder("weighted:"); - List clusterWeights = routeAction.getWeightedCluster(); - for (ClusterWeight clusterWeight : clusterWeights) { - sb.append(clusterWeight.getName()).append('_'); - } - sb.append(routeAction.hashCode()); - actionName = sb.toString(); - if (rawActions.containsKey(actionName)) { - // Just in case of hash collision, append existingActions.size() to make actionName - // unique. However, in case of collision, when new ConfigUpdate is received, actions - // and actionNames might be associated differently from the previous update, but it - // is just suboptimal and won't cause a problem. - actionName = actionName + "_" + existingActions.size(); - } - actionPolicy = generateWeightedTargetRawConfig(clusterWeights); - } - existingActions.put(routeAction, actionName); - List childPolicies = ImmutableList.of(actionPolicy); - rawActions.put(actionName, ImmutableMap.of("childPolicy", childPolicies)); - } - ImmutableMap configRoute = convertToRawRoute(route.getRouteMatch(), actionName); - rawRoutes.add(configRoute); - } - return ImmutableMap.of( - XdsLbPolicies.XDS_ROUTING_POLICY_NAME, - ImmutableMap.of( - "route", Collections.unmodifiableList(rawRoutes), - "action", Collections.unmodifiableMap(rawActions))); - } - - @VisibleForTesting - static ImmutableMap convertToRawRoute(RouteMatch routeMatch, String actionName) { - ImmutableMap.Builder configRouteBuilder = new ImmutableMap.Builder<>(); - - PathMatcher pathMatcher = routeMatch.getPathMatch(); - String path = pathMatcher.getPath(); - String prefix = pathMatcher.getPrefix(); - Pattern regex = pathMatcher.getRegEx(); - if (path != null) { - configRouteBuilder.put("path", path); - } - if (prefix != null) { - configRouteBuilder.put("prefix", prefix); - } - if (regex != null) { - configRouteBuilder.put("regex", regex.pattern()); - } - - ImmutableList.Builder rawHeaderMatcherListBuilder = new ImmutableList.Builder<>(); - List headerMatchers = routeMatch.getHeaderMatchers(); - for (HeaderMatcher headerMatcher : headerMatchers) { - ImmutableMap.Builder rawHeaderMatcherBuilder = new ImmutableMap.Builder<>(); - rawHeaderMatcherBuilder.put("name", headerMatcher.getName()); - String exactMatch = headerMatcher.getExactMatch(); - Pattern regexMatch = headerMatcher.getRegExMatch(); - HeaderMatcher.Range rangeMatch = headerMatcher.getRangeMatch(); - Boolean presentMatch = headerMatcher.getPresentMatch(); - String prefixMatch = headerMatcher.getPrefixMatch(); - String suffixMatch = headerMatcher.getSuffixMatch(); - if (exactMatch != null) { - rawHeaderMatcherBuilder.put("exactMatch", exactMatch); - } - if (regexMatch != null) { - rawHeaderMatcherBuilder.put("regexMatch", regexMatch.pattern()); - } - if (rangeMatch != null) { - rawHeaderMatcherBuilder - .put( - "rangeMatch", - ImmutableMap.of("start", rangeMatch.getStart(), "end", rangeMatch.getEnd())); - } - if (presentMatch != null) { - rawHeaderMatcherBuilder.put("presentMatch", presentMatch); - } - if (prefixMatch != null) { - rawHeaderMatcherBuilder.put("prefixMatch", prefixMatch); - } - if (suffixMatch != null) { - rawHeaderMatcherBuilder.put("suffixMatch", suffixMatch); - } - rawHeaderMatcherBuilder.put("invertMatch", headerMatcher.isInvertedMatch()); - rawHeaderMatcherListBuilder.add(rawHeaderMatcherBuilder.build()); - } - ImmutableList rawHeaderMatchers = rawHeaderMatcherListBuilder.build(); - if (!rawHeaderMatchers.isEmpty()) { - configRouteBuilder.put("headers", rawHeaderMatchers); - } - - FractionMatcher matchFraction = routeMatch.getFractionMatch(); - if (matchFraction != null) { - configRouteBuilder - .put( - "matchFraction", - ImmutableMap.of( - "numerator", matchFraction.getNumerator(), - "denominator", matchFraction.getDenominator())); - } - - configRouteBuilder.put("action", actionName); - return configRouteBuilder.build(); - } - - @VisibleForTesting - static ImmutableMap generateWeightedTargetRawConfig( - List clusterWeights) { - Map targets = new LinkedHashMap<>(); - for (ClusterWeight clusterWeight : clusterWeights) { - Map childPolicy = generateCdsRawConfig(clusterWeight.getName()); - Map weightedConfig = ImmutableMap.of( - "weight", - (double) clusterWeight.getWeight(), - "childPolicy", - ImmutableList.of(childPolicy)); - targets.put(clusterWeight.getName(), weightedConfig); - } - return ImmutableMap.of( - XdsLbPolicies.WEIGHTED_TARGET_POLICY_NAME, - ImmutableMap.of("targets", targets)); - } - - private static ImmutableMap generateCdsRawConfig(String clusterName) { - return ImmutableMap.of(XdsLbPolicies.CDS_POLICY_NAME, ImmutableMap.of("cluster", clusterName)); - } - - @Override - public void shutdown() { - logger.log(XdsLogLevel.INFO, "Shutdown"); - if (xdsClient != null) { - xdsClient = xdsClientPool.returnObject(xdsClient); - } - } } diff --git a/xds/src/main/java/io/grpc/xds/XdsNameResolver2.java b/xds/src/main/java/io/grpc/xds/XdsNameResolver2.java deleted file mode 100644 index 123ddd53f48..00000000000 --- a/xds/src/main/java/io/grpc/xds/XdsNameResolver2.java +++ /dev/null @@ -1,374 +0,0 @@ -/* - * Copyright 2019 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.xds; - -import static com.google.common.base.Preconditions.checkNotNull; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Sets; -import com.google.gson.Gson; -import io.grpc.Attributes; -import io.grpc.CallOptions; -import io.grpc.InternalConfigSelector; -import io.grpc.InternalLogId; -import io.grpc.LoadBalancer.PickSubchannelArgs; -import io.grpc.Metadata; -import io.grpc.NameResolver; -import io.grpc.Status; -import io.grpc.SynchronizationContext; -import io.grpc.internal.GrpcUtil; -import io.grpc.internal.ObjectPool; -import io.grpc.xds.Bootstrapper.BootstrapInfo; -import io.grpc.xds.EnvoyProtoData.ClusterWeight; -import io.grpc.xds.EnvoyProtoData.Route; -import io.grpc.xds.EnvoyProtoData.RouteAction; -import io.grpc.xds.ThreadSafeRandom.ThreadSafeRandomImpl; -import io.grpc.xds.XdsClient.ConfigUpdate; -import io.grpc.xds.XdsClient.ConfigWatcher; -import io.grpc.xds.XdsClient.XdsClientPoolFactory; -import io.grpc.xds.XdsLogger.XdsLogLevel; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * A {@link NameResolver} for resolving gRPC target names with "xds:" scheme. - * - *

Resolving a gRPC target involves contacting the control plane management server via xDS - * protocol to retrieve service information and produce a service config to the caller. - * - * @see XdsNameResolverProvider2 - */ -final class XdsNameResolver2 extends NameResolver { - - static final CallOptions.Key CLUSTER_SELECTION_KEY = - CallOptions.Key.create("io.grpc.xds.CLUSTER_SELECTION_KEY"); - - private final XdsLogger logger; - private final String authority; - private final ServiceConfigParser serviceConfigParser; - private final SynchronizationContext syncContext; - private final Bootstrapper bootstrapper; - private final XdsClientPoolFactory xdsClientPoolFactory; - private final ThreadSafeRandom random; - private final ConcurrentMap clusterRefs = new ConcurrentHashMap<>(); - private final ConfigSelector configSelector = new ConfigSelector(); - - private volatile List routes = Collections.emptyList(); - private Listener2 listener; - private ObjectPool xdsClientPool; - private XdsClient xdsClient; - - XdsNameResolver2(String name, - ServiceConfigParser serviceConfigParser, - SynchronizationContext syncContext, - XdsClientPoolFactory xdsClientPoolFactory) { - this(name, serviceConfigParser, syncContext, Bootstrapper.getInstance(), - xdsClientPoolFactory, ThreadSafeRandomImpl.instance); - } - - XdsNameResolver2( - String name, - ServiceConfigParser serviceConfigParser, - SynchronizationContext syncContext, - Bootstrapper bootstrapper, - XdsClientPoolFactory xdsClientPoolFactory, - ThreadSafeRandom random) { - authority = GrpcUtil.checkAuthority(checkNotNull(name, "name")); - this.serviceConfigParser = checkNotNull(serviceConfigParser, "serviceConfigParser"); - this.syncContext = checkNotNull(syncContext, "syncContext"); - this.bootstrapper = checkNotNull(bootstrapper, "bootstrapper"); - this.xdsClientPoolFactory = checkNotNull(xdsClientPoolFactory, "xdsClientPoolFactory"); - this.random = checkNotNull(random, "random"); - logger = XdsLogger.withLogId(InternalLogId.allocate("xds-resolver", name)); - logger.log(XdsLogLevel.INFO, "Created resolver for {0}", name); - } - - @Override - public String getServiceAuthority() { - return authority; - } - - @Override - public void start(Listener2 listener) { - this.listener = checkNotNull(listener, "listener"); - BootstrapInfo bootstrapInfo; - try { - bootstrapInfo = bootstrapper.readBootstrap(); - } catch (Exception e) { - listener.onError( - Status.UNAVAILABLE.withDescription("Failed to load xDS bootstrap").withCause(e)); - return; - } - xdsClientPool = xdsClientPoolFactory.newXdsClientObjectPool(bootstrapInfo); - xdsClient = xdsClientPool.getObject(); - xdsClient.watchConfigData(authority, new ConfigWatcherImpl()); - } - - @Override - public void shutdown() { - logger.log(XdsLogLevel.INFO, "Shutdown"); - if (xdsClient != null) { - xdsClient = xdsClientPool.returnObject(xdsClient); - } - } - - @VisibleForTesting - static Map generateServiceConfigWithMethodTimeoutConfig(long timeoutNano) { - String timeout = timeoutNano / 1_000_000_000.0 + "s"; - Map methodConfig = new HashMap<>(); - methodConfig.put( - "name", Collections.singletonList(Collections.emptyMap())); - methodConfig.put("timeout", timeout); - return Collections.singletonMap( - "methodConfig", Collections.singletonList(Collections.unmodifiableMap(methodConfig))); - } - - @VisibleForTesting - static Map generateServiceConfigWithLoadBalancingConfig(Collection clusters) { - Map childPolicy = new HashMap<>(); - for (String cluster : clusters) { - List>> lbPolicy = - Collections.singletonList( - Collections.singletonMap( - "cds_experimental", Collections.singletonMap("cluster", cluster))); - childPolicy.put(cluster, Collections.singletonMap("lbPolicy", lbPolicy)); - } - return Collections.singletonMap("loadBalancingConfig", - Collections.singletonList( - Collections.singletonMap( - "cluster_manager_experimental", Collections.singletonMap( - "childPolicy", Collections.unmodifiableMap(childPolicy))))); - } - - @VisibleForTesting - XdsClient getXdsClient() { - return xdsClient; - } - - private void updateResolutionResult() { - Map rawServiceConfig = - generateServiceConfigWithLoadBalancingConfig(clusterRefs.keySet()); - if (logger.isLoggable(XdsLogLevel.INFO)) { - logger.log( - XdsLogLevel.INFO, "Generated service config:\n{0}", new Gson().toJson(rawServiceConfig)); - } - ConfigOrError parsedServiceConfig = serviceConfigParser.parseServiceConfig(rawServiceConfig); - Attributes attrs = - Attributes.newBuilder() - .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) - .set(InternalConfigSelector.KEY, configSelector) - .build(); - ResolutionResult result = - ResolutionResult.newBuilder() - .setAttributes(attrs) - .setServiceConfig(parsedServiceConfig) - .build(); - listener.onResult(result); - } - - private final class ConfigSelector extends InternalConfigSelector { - @Override - public Result selectConfig(PickSubchannelArgs args) { - // Index ASCII headers by keys. - Map> asciiHeaders = new HashMap<>(); - Metadata headers = args.getHeaders(); - for (String headerName : headers.keys()) { - if (headerName.endsWith(Metadata.BINARY_HEADER_SUFFIX)) { - continue; - } - Metadata.Key key = Metadata.Key.of(headerName, Metadata.ASCII_STRING_MARSHALLER); - asciiHeaders.put(headerName, headers.getAll(key)); - } - String cluster = null; - Route selectedRoute = null; - do { - for (Route route : routes) { - if (route.getRouteMatch().matches( - "/" + args.getMethodDescriptor().getFullMethodName(), asciiHeaders)) { - selectedRoute = route; - break; - } - } - if (selectedRoute == null) { - return Result.forError( - Status.UNAVAILABLE.withDescription("Could not find xDS route matching RPC")); - } - RouteAction action = selectedRoute.getRouteAction(); - if (action.getCluster() != null) { - cluster = action.getCluster(); - } else if (action.getWeightedCluster() != null) { - int totalWeight = 0; - for (ClusterWeight weightedCluster : action.getWeightedCluster()) { - totalWeight += weightedCluster.getWeight(); - } - int select = random.nextInt(totalWeight); - int accumulator = 0; - for (ClusterWeight weightedCluster : action.getWeightedCluster()) { - accumulator += weightedCluster.getWeight(); - if (select < accumulator) { - cluster = weightedCluster.getName(); - break; - } - } - } - } while (!retainCluster(cluster)); - // TODO(chengyuanzhang): avoid service config generation and parsing for each call. - Map rawServiceConfig = - generateServiceConfigWithMethodTimeoutConfig( - selectedRoute.getRouteAction().getTimeoutNano()); - if (logger.isLoggable(XdsLogLevel.INFO)) { - logger.log(XdsLogLevel.INFO, - "Generated service config (method config):\n{0}", new Gson().toJson(rawServiceConfig)); - } - ConfigOrError parsedServiceConfig = serviceConfigParser.parseServiceConfig(rawServiceConfig); - Object config = parsedServiceConfig.getConfig(); - if (config == null) { - releaseCluster(cluster); - return Result.forError( - parsedServiceConfig.getError().augmentDescription( - "Failed to parse service config (method config)")); - } - final String finalCluster = cluster; - class SelectionCompleted implements Runnable { - @Override - public void run() { - releaseCluster(finalCluster); - } - } - - return - Result.newBuilder() - .setCallOptions(args.getCallOptions().withOption(CLUSTER_SELECTION_KEY, cluster)) - .setConfig(config) - .setCommittedCallback(new SelectionCompleted()) - .build(); - } - - private boolean retainCluster(String cluster) { - AtomicInteger refCount = clusterRefs.get(cluster); - if (refCount == null) { - return false; - } - int count; - do { - count = refCount.get(); - if (count == 0) { - return false; - } - } while (!refCount.compareAndSet(count, count + 1)); - return true; - } - - private void releaseCluster(final String cluster) { - int count = clusterRefs.get(cluster).decrementAndGet(); - if (count == 0) { - syncContext.execute(new Runnable() { - @Override - public void run() { - if (clusterRefs.get(cluster).get() == 0) { - clusterRefs.remove(cluster); - updateResolutionResult(); - } - } - }); - } - } - } - - // https://github.com/google/error-prone/issues/1767 - @SuppressWarnings("ModifyCollectionInEnhancedForLoop") - private class ConfigWatcherImpl implements ConfigWatcher { - private Set existingClusters; - - @Override - public void onConfigChanged(ConfigUpdate update) { - Set clusters = new HashSet<>(); - for (Route route : update.getRoutes()) { - RouteAction action = route.getRouteAction(); - if (action.getCluster() != null) { - clusters.add(action.getCluster()); - } else if (action.getWeightedCluster() != null) { - for (ClusterWeight weighedCluster : action.getWeightedCluster()) { - clusters.add(weighedCluster.getName()); - } - } - } - Set addedClusters = - existingClusters == null ? clusters : Sets.difference(clusters, existingClusters); - Set deletedClusters = - existingClusters == null - ? Collections.emptySet() : Sets.difference(existingClusters, clusters); - existingClusters = clusters; - boolean shouldUpdateResult = false; - for (String cluster : addedClusters) { - if (clusterRefs.containsKey(cluster)) { - clusterRefs.get(cluster).incrementAndGet(); - } else { - clusterRefs.put(cluster, new AtomicInteger(1)); - shouldUpdateResult = true; - } - } - // Update service config to include newly added clusters. - if (shouldUpdateResult) { - updateResolutionResult(); - } - // Make newly added clusters selectable by config selector and deleted clusters no longer - // selectable. - routes = update.getRoutes(); - shouldUpdateResult = false; - for (String cluster : deletedClusters) { - int count = clusterRefs.get(cluster).decrementAndGet(); - if (count == 0) { - clusterRefs.remove(cluster); - shouldUpdateResult = true; - } - } - if (shouldUpdateResult) { - updateResolutionResult(); - } - } - - @Override - public void onResourceDoesNotExist(String resourceName) { - logger.log(XdsLogLevel.INFO, "Resource {0} is unavailable", resourceName); - ConfigOrError parsedServiceConfig = - serviceConfigParser.parseServiceConfig(Collections.emptyMap()); - ResolutionResult result = - ResolutionResult.newBuilder() - .setServiceConfig(parsedServiceConfig) - // let channel take action for no config selector - .build(); - listener.onResult(result); - } - - @Override - public void onError(Status error) { - logger.log( - XdsLogLevel.WARNING, - "Received error from xDS client {0}: {1}", xdsClient, error.getDescription()); - listener.onError(error); - } - } -} diff --git a/xds/src/main/java/io/grpc/xds/XdsNameResolverProvider.java b/xds/src/main/java/io/grpc/xds/XdsNameResolverProvider.java index 4a5e2e5df1f..d00929cc38f 100644 --- a/xds/src/main/java/io/grpc/xds/XdsNameResolverProvider.java +++ b/xds/src/main/java/io/grpc/xds/XdsNameResolverProvider.java @@ -16,14 +16,26 @@ package io.grpc.xds; +import static com.google.common.base.Preconditions.checkNotNull; + import com.google.common.base.Preconditions; +import com.google.common.base.Stopwatch; +import com.google.common.base.Supplier; import io.grpc.Internal; import io.grpc.NameResolver.Args; import io.grpc.NameResolverProvider; +import io.grpc.SynchronizationContext; +import io.grpc.internal.BackoffPolicy; import io.grpc.internal.ExponentialBackoffPolicy; import io.grpc.internal.GrpcUtil; +import io.grpc.internal.ObjectPool; +import io.grpc.xds.Bootstrapper.BootstrapInfo; +import io.grpc.xds.XdsClient.RefCountedXdsClientObjectPool; import io.grpc.xds.XdsClient.XdsChannelFactory; +import io.grpc.xds.XdsClient.XdsClientFactory; +import io.grpc.xds.XdsClient.XdsClientPoolFactory; import java.net.URI; +import java.util.concurrent.ScheduledExecutorService; /** * A provider for {@link XdsNameResolver}. @@ -43,21 +55,23 @@ public final class XdsNameResolverProvider extends NameResolverProvider { @Override public XdsNameResolver newNameResolver(URI targetUri, Args args) { if (SCHEME.equals(targetUri.getScheme())) { - String targetPath = Preconditions.checkNotNull(targetUri.getPath(), "targetPath"); + String targetPath = checkNotNull(targetUri.getPath(), "targetPath"); Preconditions.checkArgument( targetPath.startsWith("/"), "the path component (%s) of the target (%s) must start with '/'", targetPath, targetUri); String name = targetPath.substring(1); - return - new XdsNameResolver( + XdsClientPoolFactory xdsClientPoolFactory = + new RefCountedXdsClientPoolFactory( name, - args, - new ExponentialBackoffPolicy.Provider(), - GrpcUtil.STOPWATCH_SUPPLIER, XdsChannelFactory.getInstance(), - Bootstrapper.getInstance()); + args.getSynchronizationContext(), args.getScheduledExecutorService(), + new ExponentialBackoffPolicy.Provider(), + GrpcUtil.STOPWATCH_SUPPLIER); + return new XdsNameResolver( + name, args.getServiceConfigParser(), + args.getSynchronizationContext(), xdsClientPoolFactory); } return null; } @@ -78,4 +92,41 @@ protected int priority() { // resolver. return 4; } + + static class RefCountedXdsClientPoolFactory implements XdsClientPoolFactory { + private final String serviceName; + private final XdsChannelFactory channelFactory; + private final SynchronizationContext syncContext; + private final ScheduledExecutorService timeService; + private final BackoffPolicy.Provider backoffPolicyProvider; + private final Supplier stopwatchSupplier; + + RefCountedXdsClientPoolFactory( + String serviceName, + XdsChannelFactory channelFactory, + SynchronizationContext syncContext, + ScheduledExecutorService timeService, + BackoffPolicy.Provider backoffPolicyProvider, + Supplier stopwatchSupplier) { + this.serviceName = checkNotNull(serviceName, "serviceName"); + this.channelFactory = checkNotNull(channelFactory, "channelFactory"); + this.syncContext = checkNotNull(syncContext, "syncContext"); + this.timeService = checkNotNull(timeService, "timeService"); + this.backoffPolicyProvider = checkNotNull(backoffPolicyProvider, "backoffPolicyProvider"); + this.stopwatchSupplier = checkNotNull(stopwatchSupplier, "stopwatchSupplier"); + } + + @Override + public ObjectPool newXdsClientObjectPool(final BootstrapInfo bootstrapInfo) { + XdsClientFactory xdsClientFactory = new XdsClientFactory() { + @Override + XdsClient createXdsClient() { + return new XdsClientImpl( + serviceName, bootstrapInfo.getServers(), channelFactory, bootstrapInfo.getNode(), + syncContext, timeService, backoffPolicyProvider, stopwatchSupplier); + } + }; + return new RefCountedXdsClientObjectPool(xdsClientFactory); + } + } } diff --git a/xds/src/main/java/io/grpc/xds/XdsNameResolverProvider2.java b/xds/src/main/java/io/grpc/xds/XdsNameResolverProvider2.java deleted file mode 100644 index 44c97ef85d3..00000000000 --- a/xds/src/main/java/io/grpc/xds/XdsNameResolverProvider2.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright 2019 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.xds; - -import static com.google.common.base.Preconditions.checkNotNull; - -import com.google.common.base.Preconditions; -import com.google.common.base.Stopwatch; -import com.google.common.base.Supplier; -import io.grpc.Internal; -import io.grpc.NameResolver.Args; -import io.grpc.NameResolverProvider; -import io.grpc.SynchronizationContext; -import io.grpc.internal.BackoffPolicy; -import io.grpc.internal.ExponentialBackoffPolicy; -import io.grpc.internal.GrpcUtil; -import io.grpc.internal.ObjectPool; -import io.grpc.xds.Bootstrapper.BootstrapInfo; -import io.grpc.xds.XdsClient.RefCountedXdsClientObjectPool; -import io.grpc.xds.XdsClient.XdsChannelFactory; -import io.grpc.xds.XdsClient.XdsClientFactory; -import io.grpc.xds.XdsClient.XdsClientPoolFactory; -import java.net.URI; -import java.util.concurrent.ScheduledExecutorService; - -/** - * A provider for {@link XdsNameResolver2}. - * - *

It resolves a target URI whose scheme is {@code "xds"}. The authority of the - * target URI is never used for current release. The path of the target URI, excluding the leading - * slash {@code '/'}, will indicate the name to use in the VHDS query. - * - *

This class should not be directly referenced in code. The resolver should be accessed - * through {@link io.grpc.NameResolverRegistry} with the URI scheme "xds". - */ -@Internal -public final class XdsNameResolverProvider2 extends NameResolverProvider { - - private static final String SCHEME = "xds"; - - @Override - public XdsNameResolver2 newNameResolver(URI targetUri, Args args) { - if (SCHEME.equals(targetUri.getScheme())) { - String targetPath = checkNotNull(targetUri.getPath(), "targetPath"); - Preconditions.checkArgument( - targetPath.startsWith("/"), - "the path component (%s) of the target (%s) must start with '/'", - targetPath, - targetUri); - String name = targetPath.substring(1); - XdsClientPoolFactory xdsClientPoolFactory = - new RefCountedXdsClientPoolFactory( - name, - XdsChannelFactory.getInstance(), - args.getSynchronizationContext(), args.getScheduledExecutorService(), - new ExponentialBackoffPolicy.Provider(), - GrpcUtil.STOPWATCH_SUPPLIER); - return new XdsNameResolver2( - name, args.getServiceConfigParser(), - args.getSynchronizationContext(), xdsClientPoolFactory); - } - return null; - } - - @Override - public String getDefaultScheme() { - return SCHEME; - } - - @Override - protected boolean isAvailable() { - return true; - } - - @Override - protected int priority() { - // Set priority value to be < 5 as we still want DNS resolver to be the primary default - // resolver. - return 4; - } - - static class RefCountedXdsClientPoolFactory implements XdsClientPoolFactory { - private final String serviceName; - private final XdsChannelFactory channelFactory; - private final SynchronizationContext syncContext; - private final ScheduledExecutorService timeService; - private final BackoffPolicy.Provider backoffPolicyProvider; - private final Supplier stopwatchSupplier; - - RefCountedXdsClientPoolFactory( - String serviceName, - XdsChannelFactory channelFactory, - SynchronizationContext syncContext, - ScheduledExecutorService timeService, - BackoffPolicy.Provider backoffPolicyProvider, - Supplier stopwatchSupplier) { - this.serviceName = checkNotNull(serviceName, "serviceName"); - this.channelFactory = checkNotNull(channelFactory, "channelFactory"); - this.syncContext = checkNotNull(syncContext, "syncContext"); - this.timeService = checkNotNull(timeService, "timeService"); - this.backoffPolicyProvider = checkNotNull(backoffPolicyProvider, "backoffPolicyProvider"); - this.stopwatchSupplier = checkNotNull(stopwatchSupplier, "stopwatchSupplier"); - } - - @Override - public ObjectPool newXdsClientObjectPool(final BootstrapInfo bootstrapInfo) { - XdsClientFactory xdsClientFactory = new XdsClientFactory() { - @Override - XdsClient createXdsClient() { - return new XdsClientImpl( - serviceName, bootstrapInfo.getServers(), channelFactory, bootstrapInfo.getNode(), - syncContext, timeService, backoffPolicyProvider, stopwatchSupplier); - } - }; - return new RefCountedXdsClientObjectPool(xdsClientFactory); - } - } -} diff --git a/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerTest.java index 3e89a8b0231..9bd7a8d08fc 100644 --- a/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerTest.java @@ -276,7 +276,7 @@ private static PickResult pickSubchannel(SubchannelPicker picker, String name) { .build(), new Metadata(), CallOptions.DEFAULT.withOption( - XdsNameResolver2.CLUSTER_SELECTION_KEY, name)); + XdsNameResolver.CLUSTER_SELECTION_KEY, name)); return picker.pickSubchannel(args); } diff --git a/xds/src/test/java/io/grpc/xds/XdsNameResolver2Test.java b/xds/src/test/java/io/grpc/xds/XdsNameResolver2Test.java deleted file mode 100644 index 31872a385ee..00000000000 --- a/xds/src/test/java/io/grpc/xds/XdsNameResolver2Test.java +++ /dev/null @@ -1,554 +0,0 @@ -/* - * Copyright 2020 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.xds; - -import static com.google.common.truth.Truth.assertThat; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; -import io.grpc.CallOptions; -import io.grpc.InternalConfigSelector; -import io.grpc.InternalConfigSelector.Result; -import io.grpc.Metadata; -import io.grpc.MethodDescriptor; -import io.grpc.MethodDescriptor.MethodType; -import io.grpc.NameResolver; -import io.grpc.NameResolver.ConfigOrError; -import io.grpc.NameResolver.ResolutionResult; -import io.grpc.NameResolver.ServiceConfigParser; -import io.grpc.Status; -import io.grpc.Status.Code; -import io.grpc.SynchronizationContext; -import io.grpc.internal.JsonParser; -import io.grpc.internal.JsonUtil; -import io.grpc.internal.ObjectPool; -import io.grpc.internal.PickSubchannelArgsImpl; -import io.grpc.testing.TestMethodDescriptors; -import io.grpc.xds.Bootstrapper.BootstrapInfo; -import io.grpc.xds.EnvoyProtoData.ClusterWeight; -import io.grpc.xds.EnvoyProtoData.Node; -import io.grpc.xds.EnvoyProtoData.Route; -import io.grpc.xds.EnvoyProtoData.RouteAction; -import io.grpc.xds.XdsClient.XdsClientPoolFactory; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; - -/** Unit tests for {@link XdsNameResolver2}. */ -// TODO(chengyuanzhang): should do tests with ManagedChannelImpl. -@RunWith(JUnit4.class) -public class XdsNameResolver2Test { - private static final String AUTHORITY = "foo.googleapis.com:80"; - @Rule - public final MockitoRule mocks = MockitoJUnit.rule(); - private final SynchronizationContext syncContext = new SynchronizationContext( - new Thread.UncaughtExceptionHandler() { - @Override - public void uncaughtException(Thread t, Throwable e) { - throw new AssertionError(e); - } - }); - private final ServiceConfigParser serviceConfigParser = new ServiceConfigParser() { - @Override - public ConfigOrError parseServiceConfig(Map rawServiceConfig) { - return ConfigOrError.fromConfig(rawServiceConfig); - } - }; - private final FakeXdsClientPoolFactory xdsClientPoolFactory = new FakeXdsClientPoolFactory(); - private final String cluster1 = "cluster-foo.googleapis.com"; - private final String cluster2 = "cluster-bar.googleapis.com"; - private final CallInfo call1 = new CallInfo("HelloService", "hi"); - private final CallInfo call2 = new CallInfo("GreetService", "bye"); - - @Mock - private ThreadSafeRandom mockRandom; - @Mock - private NameResolver.Listener2 mockListener; - @Captor - private ArgumentCaptor resolutionResultCaptor; - @Captor - ArgumentCaptor errorCaptor; - private XdsNameResolver2 resolver; - - @Before - public void setUp() { - Bootstrapper bootstrapper = new Bootstrapper() { - @Override - public BootstrapInfo readBootstrap() { - return new BootstrapInfo( - ImmutableList.of( - new ServerInfo( - "trafficdirector.googleapis.com", - ImmutableList.of(), ImmutableList.of())), - Node.newBuilder().build(), - null); - } - }; - resolver = new XdsNameResolver2(AUTHORITY, serviceConfigParser, syncContext, bootstrapper, - xdsClientPoolFactory, mockRandom); - } - - @Test - public void resolve_failToBootstrap() { - Bootstrapper bootstrapper = new Bootstrapper() { - @Override - public BootstrapInfo readBootstrap() throws IOException { - throw new IOException("Fail to read bootstrap file"); - } - }; - resolver = new XdsNameResolver2(AUTHORITY, serviceConfigParser, syncContext, bootstrapper, - xdsClientPoolFactory, mockRandom); - resolver.start(mockListener); - verify(mockListener).onError(errorCaptor.capture()); - Status error = errorCaptor.getValue(); - assertThat(error.getCode()).isEqualTo(Code.UNAVAILABLE); - assertThat(error.getDescription()).isEqualTo("Failed to load xDS bootstrap"); - assertThat(error.getCause()).hasMessageThat().isEqualTo("Fail to read bootstrap file"); - } - - @SuppressWarnings("unchecked") - @Test - public void resolve_resourceNotFound() { - resolver.start(mockListener); - FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); - xdsClient.deliverResourceNotFound(); - verify(mockListener).onResult(resolutionResultCaptor.capture()); - ResolutionResult result = resolutionResultCaptor.getValue(); - assertThat(result.getAddresses()).isEmpty(); - assertThat((Map) result.getServiceConfig().getConfig()).isEmpty(); - } - - @Test - public void resolve_encounterError() { - resolver.start(mockListener); - FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); - xdsClient.deliverError(Status.UNAVAILABLE.withDescription("server unreachable")); - verify(mockListener).onError(errorCaptor.capture()); - Status error = errorCaptor.getValue(); - assertThat(error.getCode()).isEqualTo(Code.UNAVAILABLE); - assertThat(error.getDescription()).isEqualTo("server unreachable"); - } - - @Test - public void resolve_simpleCallSucceeds() { - InternalConfigSelector configSelector = resolveToClusters(); - Result selectResult = assertCallSelectResult(call1, configSelector, cluster1, 15.0); - selectResult.getCommittedCallback().run(); - verifyNoMoreInteractions(mockListener); - } - - @Test - public void resolve_simpleCallFailedToRoute() { - InternalConfigSelector configSelector = resolveToClusters(); - CallInfo call = new CallInfo("FooService", "barMethod"); - Result selectResult = configSelector.selectConfig( - new PickSubchannelArgsImpl(call.methodDescriptor, new Metadata(), CallOptions.DEFAULT)); - Status status = selectResult.getStatus(); - assertThat(status.isOk()).isFalse(); - assertThat(status.getCode()).isEqualTo(Code.UNAVAILABLE); - assertThat(status.getDescription()).isEqualTo("Could not find xDS route matching RPC"); - verifyNoMoreInteractions(mockListener); - } - - @SuppressWarnings("unchecked") - @Test - public void resolve_resourceUpdateAfterCallStarted() { - InternalConfigSelector configSelector = resolveToClusters(); - Result selectResult = assertCallSelectResult(call1, configSelector, cluster1, 15.0); - - reset(mockListener); - FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); - xdsClient.deliverRoutes( - Arrays.asList( - new Route( - new RouteMatch(null, call1.getFullMethodNameForPath()), - new RouteAction(TimeUnit.SECONDS.toNanos(20L), "another-cluster", null)), - new Route( - new RouteMatch(null, call2.getFullMethodNameForPath()), - new RouteAction(TimeUnit.SECONDS.toNanos(15L), cluster2, null)))); - verify(mockListener).onResult(resolutionResultCaptor.capture()); - ResolutionResult result = resolutionResultCaptor.getValue(); - // Updated service config still contains cluster1 while it is removed resource. New calls no - // longer routed to cluster1. - assertServiceConfigForLoadBalancingConfig( - Arrays.asList(cluster1, cluster2, "another-cluster"), - (Map) result.getServiceConfig().getConfig()); - assertThat(result.getAttributes().get(InternalConfigSelector.KEY)) - .isSameInstanceAs(configSelector); - assertCallSelectResult(call1, configSelector, "another-cluster", 20.0); - - selectResult.getCommittedCallback().run(); // completes previous call - verify(mockListener, times(2)).onResult(resolutionResultCaptor.capture()); - result = resolutionResultCaptor.getValue(); - assertServiceConfigForLoadBalancingConfig( - Arrays.asList(cluster2, "another-cluster"), - (Map) result.getServiceConfig().getConfig()); - verifyNoMoreInteractions(mockListener); - } - - @SuppressWarnings("unchecked") - @Test - public void resolve_resourceUpdatedBeforeCallStarted() { - InternalConfigSelector configSelector = resolveToClusters(); - reset(mockListener); - FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); - xdsClient.deliverRoutes( - Arrays.asList( - new Route( - new RouteMatch(null, call1.getFullMethodNameForPath()), - new RouteAction(TimeUnit.SECONDS.toNanos(20L), "another-cluster", null)), - new Route( - new RouteMatch(null, call2.getFullMethodNameForPath()), - new RouteAction(TimeUnit.SECONDS.toNanos(15L), cluster2, null)))); - // Two consecutive service config updates: one for removing clcuster1, - // one for adding "another=cluster". - verify(mockListener, times(2)).onResult(resolutionResultCaptor.capture()); - ResolutionResult result = resolutionResultCaptor.getValue(); - assertServiceConfigForLoadBalancingConfig( - Arrays.asList(cluster2, "another-cluster"), - (Map) result.getServiceConfig().getConfig()); - assertThat(result.getAttributes().get(InternalConfigSelector.KEY)) - .isSameInstanceAs(configSelector); - assertCallSelectResult(call1, configSelector, "another-cluster", 20.0); - - verifyNoMoreInteractions(mockListener); - } - - @SuppressWarnings("unchecked") - @Test - public void resolve_raceBetweenCallAndRepeatedResourceUpdate() { - InternalConfigSelector configSelector = resolveToClusters(); - assertCallSelectResult(call1, configSelector, cluster1, 15.0); - - reset(mockListener); - FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); - xdsClient.deliverRoutes( - Arrays.asList( - new Route( - new RouteMatch(null, call1.getFullMethodNameForPath()), - new RouteAction(TimeUnit.SECONDS.toNanos(20L), "another-cluster", null)), - new Route( - new RouteMatch(null, call2.getFullMethodNameForPath()), - new RouteAction(TimeUnit.SECONDS.toNanos(15L), cluster2, null)))); - - verify(mockListener).onResult(resolutionResultCaptor.capture()); - ResolutionResult result = resolutionResultCaptor.getValue(); - assertServiceConfigForLoadBalancingConfig( - Arrays.asList(cluster1, cluster2, "another-cluster"), - (Map) result.getServiceConfig().getConfig()); - - xdsClient.deliverRoutes( - Arrays.asList( - new Route( - new RouteMatch(null, call1.getFullMethodNameForPath()), - new RouteAction(TimeUnit.SECONDS.toNanos(15L), "another-cluster", null)), - new Route( - new RouteMatch(null, call2.getFullMethodNameForPath()), - new RouteAction(TimeUnit.SECONDS.toNanos(15L), cluster2, null)))); - verifyNoMoreInteractions(mockListener); // no cluster added/deleted - assertCallSelectResult(call1, configSelector, "another-cluster", 15.0); - } - - @Test - public void resolve_raceBetweenClusterReleasedAndResourceUpdateAddBackAgain() { - InternalConfigSelector configSelector = resolveToClusters(); - Result result = assertCallSelectResult(call1, configSelector, cluster1, 15.0); - FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); - xdsClient.deliverRoutes( - Collections.singletonList( - new Route( - new RouteMatch(null, call2.getFullMethodNameForPath()), - new RouteAction(TimeUnit.SECONDS.toNanos(15L), cluster2, null)))); - xdsClient.deliverRoutes( - Arrays.asList( - new Route( - new RouteMatch(null, call1.getFullMethodNameForPath()), - new RouteAction(TimeUnit.SECONDS.toNanos(15L), cluster1, null)), - new Route( - new RouteMatch(null, call2.getFullMethodNameForPath()), - new RouteAction(TimeUnit.SECONDS.toNanos(15L), cluster2, null)))); - result.getCommittedCallback().run(); - verifyNoMoreInteractions(mockListener); - } - - @SuppressWarnings("unchecked") - @Test - public void resolve_simpleCallSucceeds_routeToWeightedCluster() { - when(mockRandom.nextInt(anyInt())).thenReturn(90, 10); - resolver.start(mockListener); - FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); - xdsClient.deliverRoutes( - Arrays.asList( - new Route( - new RouteMatch(null, call1.getFullMethodNameForPath()), - new RouteAction( - TimeUnit.SECONDS.toNanos(20L), null, - Arrays.asList( - new ClusterWeight(cluster1, 20), new ClusterWeight(cluster2, 80)))))); - verify(mockListener).onResult(resolutionResultCaptor.capture()); - ResolutionResult result = resolutionResultCaptor.getValue(); - assertThat(result.getAddresses()).isEmpty(); - assertServiceConfigForLoadBalancingConfig( - Arrays.asList(cluster1, cluster2), (Map) result.getServiceConfig().getConfig()); - assertThat(result.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL)).isNotNull(); - InternalConfigSelector configSelector = result.getAttributes().get(InternalConfigSelector.KEY); - Result selectResult = configSelector.selectConfig( - new PickSubchannelArgsImpl(call1.methodDescriptor, new Metadata(), CallOptions.DEFAULT)); - assertThat(selectResult.getStatus().isOk()).isTrue(); - assertThat(selectResult.getCallOptions().getOption(XdsNameResolver2.CLUSTER_SELECTION_KEY)) - .isEqualTo(cluster2); - assertServiceConfigForMethodConfig(20.0, (Map) selectResult.getConfig()); - - selectResult = configSelector.selectConfig( - new PickSubchannelArgsImpl(call1.methodDescriptor, new Metadata(), CallOptions.DEFAULT)); - assertThat(selectResult.getStatus().isOk()).isTrue(); - assertThat(selectResult.getCallOptions().getOption(XdsNameResolver2.CLUSTER_SELECTION_KEY)) - .isEqualTo(cluster1); - assertServiceConfigForMethodConfig(20.0, (Map) selectResult.getConfig()); - } - - @SuppressWarnings("unchecked") - private static Result assertCallSelectResult( - CallInfo call, InternalConfigSelector configSelector, String expectedCluster, - double expectedTimeoutSec) { - Result result = configSelector.selectConfig( - new PickSubchannelArgsImpl(call.methodDescriptor, new Metadata(), CallOptions.DEFAULT)); - assertThat(result.getStatus().isOk()).isTrue(); - assertThat(result.getCallOptions().getOption(XdsNameResolver2.CLUSTER_SELECTION_KEY)) - .isEqualTo(expectedCluster); - assertServiceConfigForMethodConfig(expectedTimeoutSec, (Map) result.getConfig()); - return result; - } - - @SuppressWarnings("unchecked") - private InternalConfigSelector resolveToClusters() { - resolver.start(mockListener); - FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); - xdsClient.deliverRoutes( - Arrays.asList( - new Route( - new RouteMatch(null, call1.getFullMethodNameForPath()), - new RouteAction(TimeUnit.SECONDS.toNanos(15L), cluster1, null)), - new Route( - new RouteMatch(null, call2.getFullMethodNameForPath()), - new RouteAction(TimeUnit.SECONDS.toNanos(15L), cluster2, null)))); - verify(mockListener).onResult(resolutionResultCaptor.capture()); - ResolutionResult result = resolutionResultCaptor.getValue(); - assertThat(result.getAddresses()).isEmpty(); - assertServiceConfigForLoadBalancingConfig( - Arrays.asList(cluster1, cluster2), (Map) result.getServiceConfig().getConfig()); - assertThat(result.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL)).isNotNull(); - return result.getAttributes().get(InternalConfigSelector.KEY); - } - - /** - * Verifies the raw service config contains a single method config for method with the - * specified timeout. - */ - private static void assertServiceConfigForMethodConfig( - double timeoutSec, Map actualServiceConfig) { - List> rawMethodConfigs = - JsonUtil.getListOfObjects(actualServiceConfig, "methodConfig"); - Map methodConfig = Iterables.getOnlyElement(rawMethodConfigs); - List> methods = JsonUtil.getListOfObjects(methodConfig, "name"); - assertThat(Iterables.getOnlyElement(methods)).isEmpty(); - assertThat(JsonUtil.getString(methodConfig, "timeout")).isEqualTo(timeoutSec + "s"); - } - - /** - * Verifies the raw service config contains an xDS load balancing config for the given clusters. - */ - private static void assertServiceConfigForLoadBalancingConfig( - List clusters, Map actualServiceConfig) { - List> rawLbConfigs = - JsonUtil.getListOfObjects(actualServiceConfig, "loadBalancingConfig"); - Map lbConfig = Iterables.getOnlyElement(rawLbConfigs); - assertThat(lbConfig.keySet()).containsExactly("cluster_manager_experimental"); - Map clusterManagerLbConfig = - JsonUtil.getObject(lbConfig, "cluster_manager_experimental"); - Map clusterManagerChildLbPolicies = - JsonUtil.getObject(clusterManagerLbConfig, "childPolicy"); - assertThat(clusterManagerChildLbPolicies.keySet()).containsExactlyElementsIn(clusters); - for (String cluster : clusters) { - Map childLbConfig = JsonUtil.getObject(clusterManagerChildLbPolicies, cluster); - assertThat(childLbConfig.keySet()).containsExactly("lbPolicy"); - List> childLbConfigValues = - JsonUtil.getListOfObjects(childLbConfig, "lbPolicy"); - Map cdsLbPolicy = Iterables.getOnlyElement(childLbConfigValues); - assertThat(cdsLbPolicy.keySet()).containsExactly("cds_experimental"); - assertThat(JsonUtil.getObject(cdsLbPolicy, "cds_experimental")) - .containsExactly("cluster", cluster); - } - } - - @SuppressWarnings("unchecked") - @Test - public void generateServiceConfig_forLoadBalancingConfig() throws IOException { - List clusters = Arrays.asList("cluster-foo", "cluster-bar", "cluster-baz"); - String expectedServiceConfigJson = "{\n" - + " \"loadBalancingConfig\": [{\n" - + " \"cluster_manager_experimental\": {\n" - + " \"childPolicy\": {\n" - + " \"cluster-foo\": {\n" - + " \"lbPolicy\": [{\n" - + " \"cds_experimental\": {\n" - + " \"cluster\": \"cluster-foo\"\n" - + " }\n" - + " }]\n" - + " },\n" - + " \"cluster-bar\": {\n" - + " \"lbPolicy\": [{\n" - + " \"cds_experimental\": {\n" - + " \"cluster\": \"cluster-bar\"\n" - + " }\n" - + " }]\n" - + " },\n" - + " \"cluster-baz\": {\n" - + " \"lbPolicy\": [{\n" - + " \"cds_experimental\": {\n" - + " \"cluster\": \"cluster-baz\"\n" - + " }\n" - + " }]\n" - + " }\n" - + " }\n" - + " }\n" - + " }]\n" - + "}"; - Map expectedServiceConfig = - (Map) JsonParser.parse(expectedServiceConfigJson); - assertThat(XdsNameResolver2.generateServiceConfigWithLoadBalancingConfig(clusters)) - .isEqualTo(expectedServiceConfig); - } - - @SuppressWarnings("unchecked") - @Test - public void generateServiceConfig_forMethodTimeoutConfig() throws IOException { - long timeoutNano = TimeUnit.SECONDS.toNanos(1L) + 1L; // 1.0000000001s - String expectedServiceConfigJson = "{\n" - + " \"methodConfig\": [{\n" - + " \"name\": [ {} ],\n" - + " \"timeout\": \"1.000000001s\"\n" - + " }]\n" - + "}"; - Map expectedServiceConfig = - (Map) JsonParser.parse(expectedServiceConfigJson); - assertThat(XdsNameResolver2.generateServiceConfigWithMethodTimeoutConfig(timeoutNano)) - .isEqualTo(expectedServiceConfig); - } - - private final class FakeXdsClientPoolFactory implements XdsClientPoolFactory { - @Override - public ObjectPool newXdsClientObjectPool(BootstrapInfo bootstrapInfo) { - return new ObjectPool() { - @Override - public XdsClient getObject() { - return new FakeXdsClient(); - } - - @Override - public XdsClient returnObject(Object object) { - return null; - } - }; - } - } - - private class FakeXdsClient extends XdsClient { - private String resource; - private ConfigWatcher watcher; - - @Override - void watchConfigData(String targetAuthority, ConfigWatcher watcher) { - resource = targetAuthority; - this.watcher = watcher; - } - - @Override - void shutdown() { - // no-op - } - - void deliverRoutes(final List routes) { - syncContext.execute(new Runnable() { - @Override - public void run() { - watcher.onConfigChanged(ConfigUpdate.newBuilder().addRoutes(routes).build()); - } - }); - } - - void deliverError(final Status error) { - syncContext.execute(new Runnable() { - @Override - public void run() { - watcher.onError(error); - } - }); - } - - void deliverResourceNotFound() { - Preconditions.checkState(resource != null, "no resource subscribed"); - syncContext.execute(new Runnable() { - @Override - public void run() { - watcher.onResourceDoesNotExist(resource); - } - }); - } - } - - private static final class CallInfo { - private final String service; - private final String method; - private final MethodDescriptor methodDescriptor; - - private CallInfo(String service, String method) { - this.service = service; - this.method = method; - methodDescriptor = - MethodDescriptor.newBuilder() - .setType(MethodType.UNARY).setFullMethodName(service + "/" + method) - .setRequestMarshaller(TestMethodDescriptors.voidMarshaller()) - .setResponseMarshaller(TestMethodDescriptors.voidMarshaller()) - .build(); - } - - private String getFullMethodNameForPath() { - return "/" + service + "/" + method; - } - } -} diff --git a/xds/src/test/java/io/grpc/xds/XdsNameResolverIntegrationTest.java b/xds/src/test/java/io/grpc/xds/XdsNameResolverIntegrationTest.java deleted file mode 100644 index e843fd14a40..00000000000 --- a/xds/src/test/java/io/grpc/xds/XdsNameResolverIntegrationTest.java +++ /dev/null @@ -1,570 +0,0 @@ -/* - * Copyright 2020 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.xds; - -import static com.google.common.truth.Truth.assertThat; -import static io.grpc.xds.XdsClientTestHelper.buildDiscoveryResponseV2; -import static io.grpc.xds.XdsClientTestHelper.buildListenerV2; -import static io.grpc.xds.XdsClientTestHelper.buildRouteConfigurationV2; -import static io.grpc.xds.XdsClientTestHelper.buildVirtualHostV2; -import static io.grpc.xds.XdsNameResolverTest.assertCdsPolicy; -import static io.grpc.xds.XdsNameResolverTest.assertWeightedTargetPolicy; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; -import com.google.protobuf.Any; -import com.google.protobuf.UInt32Value; -import io.envoyproxy.envoy.api.v2.DiscoveryRequest; -import io.envoyproxy.envoy.api.v2.DiscoveryResponse; -import io.envoyproxy.envoy.api.v2.core.AggregatedConfigSource; -import io.envoyproxy.envoy.api.v2.core.ConfigSource; -import io.envoyproxy.envoy.api.v2.route.Route; -import io.envoyproxy.envoy.api.v2.route.RouteAction; -import io.envoyproxy.envoy.api.v2.route.RouteMatch; -import io.envoyproxy.envoy.api.v2.route.VirtualHost; -import io.envoyproxy.envoy.api.v2.route.WeightedCluster; -import io.envoyproxy.envoy.api.v2.route.WeightedCluster.ClusterWeight; -import io.envoyproxy.envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager; -import io.envoyproxy.envoy.config.filter.network.http_connection_manager.v2.Rds; -import io.envoyproxy.envoy.service.discovery.v2.AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceImplBase; -import io.grpc.ChannelLogger; -import io.grpc.ManagedChannel; -import io.grpc.NameResolver; -import io.grpc.NameResolver.ConfigOrError; -import io.grpc.NameResolver.ResolutionResult; -import io.grpc.NameResolver.ServiceConfigParser; -import io.grpc.Status; -import io.grpc.Status.Code; -import io.grpc.SynchronizationContext; -import io.grpc.inprocess.InProcessChannelBuilder; -import io.grpc.inprocess.InProcessServerBuilder; -import io.grpc.internal.BackoffPolicy; -import io.grpc.internal.FakeClock; -import io.grpc.internal.GrpcUtil; -import io.grpc.internal.ObjectPool; -import io.grpc.stub.StreamObserver; -import io.grpc.testing.GrpcCleanupRule; -import io.grpc.xds.Bootstrapper.ServerInfo; -import io.grpc.xds.EnvoyProtoData.Node; -import io.grpc.xds.XdsClient.XdsChannel; -import io.grpc.xds.XdsClient.XdsChannelFactory; -import java.io.IOException; -import java.util.ArrayDeque; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.concurrent.TimeUnit; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.mockito.ArgumentCaptor; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; - -/** Tests for {@link XdsNameResolver} with xDS service. */ -@RunWith(JUnit4.class) -// TODO(creamsoup) use parsed service config -public class XdsNameResolverIntegrationTest { - private static final String AUTHORITY = "foo.googleapis.com:80"; - private static final Node FAKE_BOOTSTRAP_NODE = - Node.newBuilder().setId("XdsNameResolverTest").build(); - - @Rule - public final MockitoRule mocks = MockitoJUnit.rule(); - @Rule - public final GrpcCleanupRule cleanupRule = new GrpcCleanupRule(); - - private final SynchronizationContext syncContext = new SynchronizationContext( - new Thread.UncaughtExceptionHandler() { - @Override - public void uncaughtException(Thread t, Throwable e) { - throw new AssertionError(e); - } - }); - - private final FakeClock fakeClock = new FakeClock(); - private final Queue> responseObservers = new ArrayDeque<>(); - private final ServiceConfigParser serviceConfigParser = new ServiceConfigParser() { - @Override - public ConfigOrError parseServiceConfig(Map rawServiceConfig) { - return ConfigOrError.fromConfig(rawServiceConfig); - } - }; - - private final NameResolver.Args args = - NameResolver.Args.newBuilder() - .setDefaultPort(8080) - .setProxyDetector(GrpcUtil.NOOP_PROXY_DETECTOR) - .setSynchronizationContext(syncContext) - .setServiceConfigParser(serviceConfigParser) - .setScheduledExecutorService(fakeClock.getScheduledExecutorService()) - .setChannelLogger(mock(ChannelLogger.class)) - .build(); - - ArgumentCaptor resolutionResultCaptor = ArgumentCaptor.forClass(null); - - @Mock - private BackoffPolicy.Provider backoffPolicyProvider; - @Mock - private NameResolver.Listener2 mockListener; - - private XdsChannelFactory channelFactory; - private XdsNameResolver xdsNameResolver; - - @Before - public void setUp() throws IOException { - final String serverName = InProcessServerBuilder.generateName(); - AggregatedDiscoveryServiceImplBase serviceImpl = new AggregatedDiscoveryServiceImplBase() { - @Override - public StreamObserver streamAggregatedResources( - final StreamObserver responseObserver) { - responseObservers.offer(responseObserver); - @SuppressWarnings("unchecked") - StreamObserver requestObserver = mock(StreamObserver.class); - return requestObserver; - } - }; - - cleanupRule.register( - InProcessServerBuilder - .forName(serverName) - .addService(serviceImpl) - .directExecutor() - .build() - .start()); - final ManagedChannel channel = - cleanupRule.register(InProcessChannelBuilder.forName(serverName).directExecutor().build()); - - channelFactory = new XdsChannelFactory() { - @Override - XdsChannel createChannel(List servers) { - assertThat(Iterables.getOnlyElement(servers).getServerUri()).isEqualTo(serverName); - return new XdsChannel(channel, false); - } - }; - Bootstrapper bootstrapper = new Bootstrapper() { - @Override - public BootstrapInfo readBootstrap() { - List serverList = - ImmutableList.of(new ServerInfo(serverName, ImmutableList.of(), null)); - return new BootstrapInfo(serverList, FAKE_BOOTSTRAP_NODE, null); - } - }; - xdsNameResolver = - new XdsNameResolver( - AUTHORITY, - args, - backoffPolicyProvider, - fakeClock.getStopwatchSupplier(), - channelFactory, - bootstrapper); - assertThat(responseObservers).isEmpty(); - } - - @After - public void tearDown() { - xdsNameResolver.shutdown(); - } - - @Test - public void resolve_bootstrapProvidesNoTrafficDirectorInfo() { - Bootstrapper bootstrapper = new Bootstrapper() { - @Override - public BootstrapInfo readBootstrap() { - return new BootstrapInfo(ImmutableList.of(), FAKE_BOOTSTRAP_NODE, null); - } - }; - - XdsNameResolver resolver = - new XdsNameResolver( - AUTHORITY, - args, - backoffPolicyProvider, - fakeClock.getStopwatchSupplier(), - channelFactory, - bootstrapper); - resolver.start(mockListener); - ArgumentCaptor statusCaptor = ArgumentCaptor.forClass(null); - verify(mockListener).onError(statusCaptor.capture()); - assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); - assertThat(statusCaptor.getValue().getDescription()) - .isEqualTo("No management server provided by bootstrap"); - } - - @Test - public void resolve_failToBootstrap() { - Bootstrapper bootstrapper = new Bootstrapper() { - @Override - public BootstrapInfo readBootstrap() throws IOException { - throw new IOException("Fail to read bootstrap file"); - } - }; - - XdsNameResolver resolver = - new XdsNameResolver( - AUTHORITY, - args, - backoffPolicyProvider, - fakeClock.getStopwatchSupplier(), - channelFactory, - bootstrapper); - resolver.start(mockListener); - ArgumentCaptor errorCaptor = ArgumentCaptor.forClass(null); - verify(mockListener).onError(errorCaptor.capture()); - Status error = errorCaptor.getValue(); - assertThat(error.getCode()).isEqualTo(Code.UNAVAILABLE); - assertThat(error.getDescription()).isEqualTo("Failed to bootstrap"); - assertThat(error.getCause()).hasMessageThat().isEqualTo("Fail to read bootstrap file"); - } - - @Test - public void resolve_passXdsClientPoolInResult() { - xdsNameResolver.start(mockListener); - assertThat(responseObservers).hasSize(1); - StreamObserver responseObserver = responseObservers.poll(); - - // Simulate receiving an LDS response that contains cluster resolution directly in-line. - String clusterName = "cluster-foo.googleapis.com"; - responseObserver.onNext( - buildLdsResponseForCluster("0", AUTHORITY, clusterName, "0000")); - - verify(mockListener).onResult(resolutionResultCaptor.capture()); - ResolutionResult result = resolutionResultCaptor.getValue(); - ObjectPool xdsClientPool = result.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL); - assertThat(xdsClientPool).isNotNull(); - } - - @SuppressWarnings("unchecked") - @Test - public void resolve_ResourceNotFound() { - xdsNameResolver.start(mockListener); - assertThat(responseObservers).hasSize(1); - StreamObserver responseObserver = responseObservers.poll(); - - // Simulate receiving an LDS response that does not contain requested resource. - String clusterName = "cluster-bar.googleapis.com"; - responseObserver.onNext( - buildLdsResponseForCluster("0", "bar.googleapis.com", clusterName, "0000")); - - fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); - verify(mockListener).onResult(resolutionResultCaptor.capture()); - ResolutionResult result = resolutionResultCaptor.getValue(); - assertThat(result.getAddresses()).isEmpty(); - assertThat((Map) result.getServiceConfig().getConfig()).isEmpty(); - } - - @Test - @SuppressWarnings("unchecked") - public void resolve_resourceUpdated() { - xdsNameResolver.start(mockListener); - assertThat(responseObservers).hasSize(1); - StreamObserver responseObserver = responseObservers.poll(); - - // Simulate receiving an LDS response that contains cluster resolution directly in-line. - responseObserver.onNext( - buildLdsResponseForCluster("0", AUTHORITY, "cluster-foo.googleapis.com", "0000")); - - verify(mockListener).onResult(resolutionResultCaptor.capture()); - ResolutionResult result = resolutionResultCaptor.getValue(); - assertThat(result.getAddresses()).isEmpty(); - Map serviceConfig = (Map) result.getServiceConfig().getConfig(); - - List> rawLbConfigs = - (List>) serviceConfig.get("loadBalancingConfig"); - Map lbConfig = Iterables.getOnlyElement(rawLbConfigs); - assertThat(lbConfig.keySet()).containsExactly("xds_routing_experimental"); - Map rawConfigValues = (Map) lbConfig.get("xds_routing_experimental"); - Map> actions = - (Map>) rawConfigValues.get("action"); - List> routes = (List>) rawConfigValues.get("route"); - Map route = Iterables.getOnlyElement(routes); - assertThat(route.keySet()).containsExactly("prefix", "action"); - assertThat((String) route.get("prefix")).isEqualTo(""); - assertCdsPolicy(actions.get(route.get("action")), "cluster-foo.googleapis.com"); - - // Simulate receiving another LDS response that tells client to do RDS. - String routeConfigName = "route-foo.googleapis.com"; - responseObserver.onNext( - buildLdsResponseForRdsResource("1", AUTHORITY, routeConfigName, "0001")); - - // Client sent an RDS request for resource "route-foo.googleapis.com" (Omitted in this test). - - // Simulate receiving an RDS response that contains the resource "route-foo.googleapis.com" - // with cluster resolution for "foo.googleapis.com". - responseObserver.onNext( - buildRdsResponseForCluster("0", routeConfigName, AUTHORITY, - "cluster-blade.googleapis.com", "0000")); - - verify(mockListener, times(2)).onResult(resolutionResultCaptor.capture()); - result = resolutionResultCaptor.getValue(); - assertThat(result.getAddresses()).isEmpty(); - serviceConfig = (Map) result.getServiceConfig().getConfig(); - rawLbConfigs = (List>) serviceConfig.get("loadBalancingConfig"); - lbConfig = Iterables.getOnlyElement(rawLbConfigs); - assertThat(lbConfig.keySet()).containsExactly("xds_routing_experimental"); - rawConfigValues = (Map) lbConfig.get("xds_routing_experimental"); - actions = (Map>) rawConfigValues.get("action"); - routes = (List>) rawConfigValues.get("route"); - route = Iterables.getOnlyElement(routes); - assertThat(route.keySet()).containsExactly("prefix", "action"); - assertThat((String) route.get("prefix")).isEqualTo(""); - assertCdsPolicy(actions.get(route.get("action")), "cluster-blade.googleapis.com"); - } - - @Test - @SuppressWarnings("unchecked") - public void resolve_xdsRoutingLoadBalancing() { - xdsNameResolver.start(mockListener); - assertThat(responseObservers).hasSize(1); - StreamObserver responseObserver = responseObservers.poll(); - - // Simulate receiving an LDS response that contains routes resolution directly in-line. - List protoRoutes = - ImmutableList.of( - // path match, routed to cluster - Route.newBuilder() - .setMatch(RouteMatch.newBuilder().setPath("/fooSvc/hello")) - .setRoute(buildClusterRoute("cluster-hello.googleapis.com")) - .build(), - // prefix match, routed to cluster - Route.newBuilder() - .setMatch(RouteMatch.newBuilder().setPrefix("/fooSvc/")) - .setRoute(buildClusterRoute("cluster-foo.googleapis.com")) - .build(), - // path match, routed to weighted clusters - Route.newBuilder() - .setMatch(RouteMatch.newBuilder().setPath("/barSvc/hello")) - .setRoute(buildWeightedClusterRoute(ImmutableMap.of( - "cluster-hello.googleapis.com", 40, "cluster-hello2.googleapis.com", 60))) - .build(), - // prefix match, routed to weighted clusters - Route.newBuilder() - .setMatch(RouteMatch.newBuilder().setPrefix("/barSvc/")) - .setRoute( - buildWeightedClusterRoute( - ImmutableMap.of( - "cluster-bar.googleapis.com", 30, "cluster-bar2.googleapis.com", 70))) - .build(), - // default with prefix = "/", routed to cluster - Route.newBuilder() - .setMatch(RouteMatch.newBuilder().setPrefix("/")) - .setRoute(buildClusterRoute("cluster-hello.googleapis.com")) - .build()); - HttpConnectionManager httpConnectionManager = - HttpConnectionManager.newBuilder() - .setRouteConfig( - buildRouteConfigurationV2( - "route-foo.googleapis.com", // doesn't matter - ImmutableList.of(buildVirtualHostForRoutes(AUTHORITY, protoRoutes)))) - .build(); - List listeners = - ImmutableList.of(Any.pack(buildListenerV2(AUTHORITY, Any.pack(httpConnectionManager)))); - responseObserver.onNext( - buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000")); - - verify(mockListener).onResult(resolutionResultCaptor.capture()); - ResolutionResult result = resolutionResultCaptor.getValue(); - assertThat(result.getAddresses()).isEmpty(); - Map serviceConfig = (Map) result.getServiceConfig().getConfig(); - - List> rawLbConfigs = - (List>) serviceConfig.get("loadBalancingConfig"); - Map lbConfig = Iterables.getOnlyElement(rawLbConfigs); - assertThat(lbConfig.keySet()).containsExactly("xds_routing_experimental"); - Map rawConfigValues = (Map) lbConfig.get("xds_routing_experimental"); - assertThat(rawConfigValues.keySet()).containsExactly("action", "route"); - Map> actions = - (Map>) rawConfigValues.get("action"); - List> routes = (List>) rawConfigValues.get("route"); - assertThat(actions).hasSize(4); - assertThat(routes).hasSize(5); - - Map route0 = routes.get(0); - assertThat(route0.keySet()).containsExactly("path", "action"); - assertThat((String) route0.get("path")).isEqualTo("/fooSvc/hello"); - assertCdsPolicy(actions.get(route0.get("action")), "cluster-hello.googleapis.com"); - - Map route1 = routes.get(1); - assertThat(route1.keySet()).containsExactly("prefix", "action"); - assertThat((String) route1.get("prefix")).isEqualTo("/fooSvc/"); - assertCdsPolicy(actions.get(route1.get("action")), "cluster-foo.googleapis.com"); - - Map route2 = routes.get(2); - assertThat(route2.keySet()).containsExactly("path", "action"); - assertThat((String) route2.get("path")).isEqualTo("/barSvc/hello"); - assertWeightedTargetPolicy( - actions.get(route2.get("action")), - ImmutableMap.of( - "cluster-hello.googleapis.com", 40, "cluster-hello2.googleapis.com", 60)); - - Map route3 = routes.get(3); - assertThat(route3.keySet()).containsExactly("prefix", "action"); - assertThat((String) route3.get("prefix")).isEqualTo("/barSvc/"); - assertWeightedTargetPolicy( - actions.get(route3.get("action")), - ImmutableMap.of( - "cluster-bar.googleapis.com", 30, "cluster-bar2.googleapis.com", 70)); - - Map route4 = routes.get(4); - assertThat(route4.keySet()).containsExactly("prefix", "action"); - assertThat((String) route4.get("prefix")).isEqualTo("/"); - assertCdsPolicy(actions.get(route4.get("action")), "cluster-hello.googleapis.com"); - } - - @Test - @SuppressWarnings("unchecked") - public void resolve_resourceNewlyAdded() { - xdsNameResolver.start(mockListener); - assertThat(responseObservers).hasSize(1); - StreamObserver responseObserver = responseObservers.poll(); - - // Simulate receiving an LDS response that does not contain requested resource. - responseObserver.onNext( - buildLdsResponseForCluster("0", "bar.googleapis.com", - "cluster-bar.googleapis.com", "0000")); - - fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); - verify(mockListener).onResult(resolutionResultCaptor.capture()); - ResolutionResult result = resolutionResultCaptor.getValue(); - assertThat(result.getAddresses()).isEmpty(); - assertThat((Map) result.getServiceConfig().getConfig()).isEmpty(); - - // Simulate receiving another LDS response that contains cluster resolution directly in-line. - responseObserver.onNext( - buildLdsResponseForCluster("1", AUTHORITY, "cluster-foo.googleapis.com", - "0001")); - - verify(mockListener, times(2)).onResult(resolutionResultCaptor.capture()); - result = resolutionResultCaptor.getValue(); - assertThat(result.getAddresses()).isEmpty(); - Map serviceConfig = (Map) result.getServiceConfig().getConfig(); - List> rawLbConfigs = - (List>) serviceConfig.get("loadBalancingConfig"); - Map lbConfig = Iterables.getOnlyElement(rawLbConfigs); - assertThat(lbConfig.keySet()).containsExactly("xds_routing_experimental"); - Map rawConfigValues = (Map) lbConfig.get("xds_routing_experimental"); - Map> actions = - (Map>) rawConfigValues.get("action"); - List> routes = (List>) rawConfigValues.get("route"); - Map route = Iterables.getOnlyElement(routes); - assertThat(route.keySet()).containsExactly("prefix", "action"); - assertThat((String) route.get("prefix")).isEqualTo(""); - assertCdsPolicy(actions.get(route.get("action")), "cluster-foo.googleapis.com"); - } - - /** - * Builds an LDS DiscoveryResponse containing the mapping of given host to - * the given cluster name directly in-line. Clients receiving this response is - * able to resolve cluster name for the given host immediately. - */ - private static DiscoveryResponse buildLdsResponseForCluster( - String versionInfo, String host, String clusterName, String nonce) { - List listeners = ImmutableList.of( - Any.pack(buildListenerV2(host, // target Listener resource - Any.pack( - HttpConnectionManager.newBuilder() - .setRouteConfig( - buildRouteConfigurationV2("route-foo.googleapis.com", // doesn't matter - ImmutableList.of( - buildVirtualHostV2( - ImmutableList.of(host), // exact match - clusterName)))) - .build())))); - return buildDiscoveryResponseV2( - versionInfo, listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, nonce); - } - - /** - * Builds an LDS DiscoveryResponse containing the mapping of given host to - * the given RDS resource name. Clients receiving this response is able to - * send an RDS request for resolving the cluster name for the given host. - */ - private static DiscoveryResponse buildLdsResponseForRdsResource( - String versionInfo, String host, String routeConfigName, String nonce) { - Rds rdsConfig = - Rds.newBuilder() - // Must set to use ADS. - .setConfigSource( - ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) - .setRouteConfigName(routeConfigName) - .build(); - - List listeners = ImmutableList.of( - Any.pack( - buildListenerV2( - host, Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build())))); - return buildDiscoveryResponseV2( - versionInfo, listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, nonce); - } - - /** - * Builds an RDS DiscoveryResponse containing route configuration with the given name and a - * virtual host that matches the given host to the given cluster name. - */ - private static DiscoveryResponse buildRdsResponseForCluster( - String versionInfo, - String routeConfigName, - String host, - String clusterName, - String nonce) { - List routeConfigs = ImmutableList.of( - Any.pack( - buildRouteConfigurationV2( - routeConfigName, - ImmutableList.of( - buildVirtualHostV2(ImmutableList.of(host), clusterName))))); - return buildDiscoveryResponseV2( - versionInfo, routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, nonce); - } - - private static RouteAction buildClusterRoute(String clusterName) { - return RouteAction.newBuilder().setCluster(clusterName).build(); - } - - /** - * Builds a RouteAction for a weighted cluster route. The given map is keyed by cluster name and - * valued by the weight of the cluster. - */ - private static RouteAction buildWeightedClusterRoute(Map clusterWeights) { - WeightedCluster.Builder builder = WeightedCluster.newBuilder(); - for (Map.Entry entry : clusterWeights.entrySet()) { - builder.addClusters( - ClusterWeight.newBuilder() - .setName(entry.getKey()) - .setWeight(UInt32Value.of(entry.getValue()))); - } - return RouteAction.newBuilder() - .setWeightedClusters(builder) - .build(); - } - - private static VirtualHost buildVirtualHostForRoutes(String domain, List routes) { - return VirtualHost.newBuilder() - .setName("virtualhost00.googleapis.com") // don't care - .addAllDomains(ImmutableList.of(domain)) - .addAllRoutes(routes) - .build(); - } -} diff --git a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java index 16ce37a6e51..b1142380a6f 100644 --- a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java @@ -17,264 +17,538 @@ package io.grpc.xds; import static com.google.common.truth.Truth.assertThat; -import static io.grpc.xds.XdsLbPolicies.CDS_POLICY_NAME; -import static io.grpc.xds.XdsLbPolicies.WEIGHTED_TARGET_POLICY_NAME; -import static io.grpc.xds.XdsLbPolicies.XDS_ROUTING_POLICY_NAME; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; -import com.google.common.collect.ImmutableMap; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; -import com.google.re2j.Pattern; +import io.grpc.CallOptions; +import io.grpc.InternalConfigSelector; +import io.grpc.InternalConfigSelector.Result; +import io.grpc.Metadata; +import io.grpc.MethodDescriptor; +import io.grpc.MethodDescriptor.MethodType; +import io.grpc.NameResolver; +import io.grpc.NameResolver.ConfigOrError; +import io.grpc.NameResolver.ResolutionResult; +import io.grpc.NameResolver.ServiceConfigParser; +import io.grpc.Status; +import io.grpc.Status.Code; +import io.grpc.SynchronizationContext; import io.grpc.internal.JsonParser; +import io.grpc.internal.JsonUtil; +import io.grpc.internal.ObjectPool; +import io.grpc.internal.PickSubchannelArgsImpl; +import io.grpc.testing.TestMethodDescriptors; +import io.grpc.xds.Bootstrapper.BootstrapInfo; import io.grpc.xds.EnvoyProtoData.ClusterWeight; +import io.grpc.xds.EnvoyProtoData.Node; import io.grpc.xds.EnvoyProtoData.Route; import io.grpc.xds.EnvoyProtoData.RouteAction; -import io.grpc.xds.RouteMatch.FractionMatcher; -import io.grpc.xds.RouteMatch.HeaderMatcher; -import io.grpc.xds.RouteMatch.PathMatcher; +import io.grpc.xds.XdsClient.XdsClientPoolFactory; import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; /** Unit tests for {@link XdsNameResolver}. */ +// TODO(chengyuanzhang): should do tests with ManagedChannelImpl. @RunWith(JUnit4.class) public class XdsNameResolverTest { + private static final String AUTHORITY = "foo.googleapis.com:80"; + @Rule + public final MockitoRule mocks = MockitoJUnit.rule(); + private final SynchronizationContext syncContext = new SynchronizationContext( + new Thread.UncaughtExceptionHandler() { + @Override + public void uncaughtException(Thread t, Throwable e) { + throw new AssertionError(e); + } + }); + private final ServiceConfigParser serviceConfigParser = new ServiceConfigParser() { + @Override + public ConfigOrError parseServiceConfig(Map rawServiceConfig) { + return ConfigOrError.fromConfig(rawServiceConfig); + } + }; + private final FakeXdsClientPoolFactory xdsClientPoolFactory = new FakeXdsClientPoolFactory(); + private final String cluster1 = "cluster-foo.googleapis.com"; + private final String cluster2 = "cluster-bar.googleapis.com"; + private final CallInfo call1 = new CallInfo("HelloService", "hi"); + private final CallInfo call2 = new CallInfo("GreetService", "bye"); + + @Mock + private ThreadSafeRandom mockRandom; + @Mock + private NameResolver.Listener2 mockListener; + @Captor + private ArgumentCaptor resolutionResultCaptor; + @Captor + ArgumentCaptor errorCaptor; + private XdsNameResolver resolver; + + @Before + public void setUp() { + Bootstrapper bootstrapper = new Bootstrapper() { + @Override + public BootstrapInfo readBootstrap() { + return new BootstrapInfo( + ImmutableList.of( + new ServerInfo( + "trafficdirector.googleapis.com", + ImmutableList.of(), ImmutableList.of())), + Node.newBuilder().build(), + null); + } + }; + resolver = new XdsNameResolver(AUTHORITY, serviceConfigParser, syncContext, bootstrapper, + xdsClientPoolFactory, mockRandom); + } @Test - public void generateWeightedTargetRawConfig() throws IOException { - List clusterWeights = + public void resolve_failToBootstrap() { + Bootstrapper bootstrapper = new Bootstrapper() { + @Override + public BootstrapInfo readBootstrap() throws IOException { + throw new IOException("Fail to read bootstrap file"); + } + }; + resolver = new XdsNameResolver(AUTHORITY, serviceConfigParser, syncContext, bootstrapper, + xdsClientPoolFactory, mockRandom); + resolver.start(mockListener); + verify(mockListener).onError(errorCaptor.capture()); + Status error = errorCaptor.getValue(); + assertThat(error.getCode()).isEqualTo(Code.UNAVAILABLE); + assertThat(error.getDescription()).isEqualTo("Failed to load xDS bootstrap"); + assertThat(error.getCause()).hasMessageThat().isEqualTo("Fail to read bootstrap file"); + } + + @SuppressWarnings("unchecked") + @Test + public void resolve_resourceNotFound() { + resolver.start(mockListener); + FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); + xdsClient.deliverResourceNotFound(); + verify(mockListener).onResult(resolutionResultCaptor.capture()); + ResolutionResult result = resolutionResultCaptor.getValue(); + assertThat(result.getAddresses()).isEmpty(); + assertThat((Map) result.getServiceConfig().getConfig()).isEmpty(); + } + + @Test + public void resolve_encounterError() { + resolver.start(mockListener); + FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); + xdsClient.deliverError(Status.UNAVAILABLE.withDescription("server unreachable")); + verify(mockListener).onError(errorCaptor.capture()); + Status error = errorCaptor.getValue(); + assertThat(error.getCode()).isEqualTo(Code.UNAVAILABLE); + assertThat(error.getDescription()).isEqualTo("server unreachable"); + } + + @Test + public void resolve_simpleCallSucceeds() { + InternalConfigSelector configSelector = resolveToClusters(); + Result selectResult = assertCallSelectResult(call1, configSelector, cluster1, 15.0); + selectResult.getCommittedCallback().run(); + verifyNoMoreInteractions(mockListener); + } + + @Test + public void resolve_simpleCallFailedToRoute() { + InternalConfigSelector configSelector = resolveToClusters(); + CallInfo call = new CallInfo("FooService", "barMethod"); + Result selectResult = configSelector.selectConfig( + new PickSubchannelArgsImpl(call.methodDescriptor, new Metadata(), CallOptions.DEFAULT)); + Status status = selectResult.getStatus(); + assertThat(status.isOk()).isFalse(); + assertThat(status.getCode()).isEqualTo(Code.UNAVAILABLE); + assertThat(status.getDescription()).isEqualTo("Could not find xDS route matching RPC"); + verifyNoMoreInteractions(mockListener); + } + + @SuppressWarnings("unchecked") + @Test + public void resolve_resourceUpdateAfterCallStarted() { + InternalConfigSelector configSelector = resolveToClusters(); + Result selectResult = assertCallSelectResult(call1, configSelector, cluster1, 15.0); + + reset(mockListener); + FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); + xdsClient.deliverRoutes( Arrays.asList( - new ClusterWeight("cluster-foo", 30), new ClusterWeight("cluster-bar", 50)); - Map config = XdsNameResolver.generateWeightedTargetRawConfig(clusterWeights); - String expectedJson = "{\n" - + " \"weighted_target_experimental\": {\n" - + " \"targets\": {\n" - + " \"cluster-foo\": {\n" - + " \"weight\": 30,\n" - + " \"childPolicy\": [{\n" - + " \"cds_experimental\": {\n" - + " \"cluster\": \"cluster-foo\"\n" - + " }\n" - + " }]\n" - + " },\n" - + " \"cluster-bar\": {\n" - + " \"weight\": 50,\n" - + " \"childPolicy\": [{\n" - + " \"cds_experimental\": {\n" - + " \"cluster\": \"cluster-bar\"\n" - + " }\n" - + " }]\n" - + " }\n" - + " }\n" - + " }\n" - + "}"; - assertThat(config).isEqualTo(JsonParser.parse(expectedJson)); + new Route( + new RouteMatch(null, call1.getFullMethodNameForPath()), + new RouteAction(TimeUnit.SECONDS.toNanos(20L), "another-cluster", null)), + new Route( + new RouteMatch(null, call2.getFullMethodNameForPath()), + new RouteAction(TimeUnit.SECONDS.toNanos(15L), cluster2, null)))); + verify(mockListener).onResult(resolutionResultCaptor.capture()); + ResolutionResult result = resolutionResultCaptor.getValue(); + // Updated service config still contains cluster1 while it is removed resource. New calls no + // longer routed to cluster1. + assertServiceConfigForLoadBalancingConfig( + Arrays.asList(cluster1, cluster2, "another-cluster"), + (Map) result.getServiceConfig().getConfig()); + assertThat(result.getAttributes().get(InternalConfigSelector.KEY)) + .isSameInstanceAs(configSelector); + assertCallSelectResult(call1, configSelector, "another-cluster", 20.0); + + selectResult.getCommittedCallback().run(); // completes previous call + verify(mockListener, times(2)).onResult(resolutionResultCaptor.capture()); + result = resolutionResultCaptor.getValue(); + assertServiceConfigForLoadBalancingConfig( + Arrays.asList(cluster2, "another-cluster"), + (Map) result.getServiceConfig().getConfig()); + verifyNoMoreInteractions(mockListener); } @SuppressWarnings("unchecked") @Test - public void generateXdsRoutingRawConfig() { - Route r1 = - new Route( - new RouteMatch( - new PathMatcher(null, "", null), Collections.emptyList(), - new FractionMatcher(10, 20)), - new RouteAction(15L, "cluster-foo", null)); - Route r2 = - new Route( - new RouteMatch( - new PathMatcher("/service/method", null, null), - Arrays.asList( - new HeaderMatcher(":scheme", "https", null, null, null, null, null, false)), - null), - new RouteAction( - 15L, - null, - Arrays.asList( - new ClusterWeight("cluster-foo", 20), - new ClusterWeight("cluster-bar", 20)))); - - Map config = - XdsNameResolver.generateXdsRoutingRawConfig(Arrays.asList(r1, r2)); - assertThat(config.keySet()).containsExactly("xds_routing_experimental"); - Map content = (Map) config.get(XDS_ROUTING_POLICY_NAME); - assertThat(content.keySet()).containsExactly("action", "route"); - Map> actions = (Map>) content.get("action"); - List> routes = (List>) content.get("route"); - assertThat(actions).hasSize(2); - assertThat(routes).hasSize(2); - - Map route0 = routes.get(0); - assertThat(route0.keySet()).containsExactly("prefix", "matchFraction", "action"); - assertThat((String) route0.get("prefix")).isEqualTo(""); - assertThat((Map) route0.get("matchFraction")) - .containsExactly("numerator", 10, "denominator", 20); - assertCdsPolicy(actions.get(route0.get("action")), "cluster-foo"); - - Map route1 = routes.get(1); - assertThat(route1.keySet()).containsExactly("path", "headers", "action"); - assertThat((String) route1.get("path")).isEqualTo("/service/method"); - Map header = Iterables.getOnlyElement((List>) route1.get("headers")); - assertThat(header) - .containsExactly("name", ":scheme", "exactMatch", "https", "invertMatch", false); - assertWeightedTargetPolicy( - actions.get(route1.get("action")), - ImmutableMap.of( - "cluster-foo", 20, "cluster-bar", 20)); + public void resolve_resourceUpdatedBeforeCallStarted() { + InternalConfigSelector configSelector = resolveToClusters(); + reset(mockListener); + FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); + xdsClient.deliverRoutes( + Arrays.asList( + new Route( + new RouteMatch(null, call1.getFullMethodNameForPath()), + new RouteAction(TimeUnit.SECONDS.toNanos(20L), "another-cluster", null)), + new Route( + new RouteMatch(null, call2.getFullMethodNameForPath()), + new RouteAction(TimeUnit.SECONDS.toNanos(15L), cluster2, null)))); + // Two consecutive service config updates: one for removing clcuster1, + // one for adding "another=cluster". + verify(mockListener, times(2)).onResult(resolutionResultCaptor.capture()); + ResolutionResult result = resolutionResultCaptor.getValue(); + assertServiceConfigForLoadBalancingConfig( + Arrays.asList(cluster2, "another-cluster"), + (Map) result.getServiceConfig().getConfig()); + assertThat(result.getAttributes().get(InternalConfigSelector.KEY)) + .isSameInstanceAs(configSelector); + assertCallSelectResult(call1, configSelector, "another-cluster", 20.0); + + verifyNoMoreInteractions(mockListener); } @SuppressWarnings("unchecked") @Test - public void generateXdsRoutingRawConfig_allowDuplicateMatchers() { - Route route = - new Route( - new RouteMatch( - new PathMatcher("/service/method", null, null), - Collections.emptyList(), null), - new RouteAction(15L, "cluster-foo", null)); - - Map config = - XdsNameResolver.generateXdsRoutingRawConfig(Arrays.asList(route, route)); - assertThat(config.keySet()).containsExactly(XDS_ROUTING_POLICY_NAME); - Map content = (Map) config.get(XDS_ROUTING_POLICY_NAME); - assertThat(content.keySet()).containsExactly("action", "route"); - Map actions = (Map) content.get("action"); - List routes = (List) content.get("route"); - assertThat(actions).hasSize(1); - assertThat(routes).hasSize(2); - assertThat(routes.get(0)).isEqualTo(routes.get(1)); + public void resolve_raceBetweenCallAndRepeatedResourceUpdate() { + InternalConfigSelector configSelector = resolveToClusters(); + assertCallSelectResult(call1, configSelector, cluster1, 15.0); + + reset(mockListener); + FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); + xdsClient.deliverRoutes( + Arrays.asList( + new Route( + new RouteMatch(null, call1.getFullMethodNameForPath()), + new RouteAction(TimeUnit.SECONDS.toNanos(20L), "another-cluster", null)), + new Route( + new RouteMatch(null, call2.getFullMethodNameForPath()), + new RouteAction(TimeUnit.SECONDS.toNanos(15L), cluster2, null)))); + + verify(mockListener).onResult(resolutionResultCaptor.capture()); + ResolutionResult result = resolutionResultCaptor.getValue(); + assertServiceConfigForLoadBalancingConfig( + Arrays.asList(cluster1, cluster2, "another-cluster"), + (Map) result.getServiceConfig().getConfig()); + + xdsClient.deliverRoutes( + Arrays.asList( + new Route( + new RouteMatch(null, call1.getFullMethodNameForPath()), + new RouteAction(TimeUnit.SECONDS.toNanos(15L), "another-cluster", null)), + new Route( + new RouteMatch(null, call2.getFullMethodNameForPath()), + new RouteAction(TimeUnit.SECONDS.toNanos(15L), cluster2, null)))); + verifyNoMoreInteractions(mockListener); // no cluster added/deleted + assertCallSelectResult(call1, configSelector, "another-cluster", 15.0); + } + + @Test + public void resolve_raceBetweenClusterReleasedAndResourceUpdateAddBackAgain() { + InternalConfigSelector configSelector = resolveToClusters(); + Result result = assertCallSelectResult(call1, configSelector, cluster1, 15.0); + FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); + xdsClient.deliverRoutes( + Collections.singletonList( + new Route( + new RouteMatch(null, call2.getFullMethodNameForPath()), + new RouteAction(TimeUnit.SECONDS.toNanos(15L), cluster2, null)))); + xdsClient.deliverRoutes( + Arrays.asList( + new Route( + new RouteMatch(null, call1.getFullMethodNameForPath()), + new RouteAction(TimeUnit.SECONDS.toNanos(15L), cluster1, null)), + new Route( + new RouteMatch(null, call2.getFullMethodNameForPath()), + new RouteAction(TimeUnit.SECONDS.toNanos(15L), cluster2, null)))); + result.getCommittedCallback().run(); + verifyNoMoreInteractions(mockListener); } @SuppressWarnings("unchecked") @Test - public void convertToRawRoute() throws IOException { - RouteMatch routeMatch1 = - new RouteMatch( - new PathMatcher("/service/method", null, null), - Collections.emptyList(), null); - String expectedJson1 = "{\n" - + " \"path\": \"/service/method\",\n" - + " \"action\": \"action_foo\"" - + "}"; - assertThat(XdsNameResolver.convertToRawRoute(routeMatch1, "action_foo")) - .isEqualTo(JsonParser.parse(expectedJson1)); - - RouteMatch routeMatch2 = - new RouteMatch( - new PathMatcher(null, "/", null), Collections.emptyList(), - new FractionMatcher(10, 100)); - Map rawRoute2 = XdsNameResolver.convertToRawRoute(routeMatch2, "action_foo"); - Map rawMatchFraction = (Map) rawRoute2.get("matchFraction"); - assertThat(rawMatchFraction).containsExactly("numerator", 10, "denominator", 100); - - RouteMatch routeMatch3 = - new RouteMatch( - new PathMatcher(null, "/", null), - Arrays.asList( - new HeaderMatcher("timeout", null, null, new HeaderMatcher.Range(0L, 10L), - null, null, null, false)), - null); - Map rawRoute3 = XdsNameResolver.convertToRawRoute(routeMatch3, "action_foo"); - Map header = - (Map) Iterables.getOnlyElement((List) rawRoute3.get("headers")); - assertThat((Map) header.get("rangeMatch")).containsExactly("start", 0L, "end", 10L); - - RouteMatch routeMatch4 = - new RouteMatch( - new PathMatcher(null, "/", null), - Arrays.asList( - new HeaderMatcher(":scheme", "https", null, null, null, null, null, false), - new HeaderMatcher( - ":path", null, Pattern.compile("google.*"), null, null, null, null, true), - new HeaderMatcher("timeout", null, null, null, true, null, null, false), - new HeaderMatcher(":authority", null, null, null, null, "google", null, false), - new HeaderMatcher(":authority", null, null, null, null, null, "grpc.io", false)), - null); + public void resolve_simpleCallSucceeds_routeToWeightedCluster() { + when(mockRandom.nextInt(anyInt())).thenReturn(90, 10); + resolver.start(mockListener); + FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); + xdsClient.deliverRoutes( + Arrays.asList( + new Route( + new RouteMatch(null, call1.getFullMethodNameForPath()), + new RouteAction( + TimeUnit.SECONDS.toNanos(20L), null, + Arrays.asList( + new ClusterWeight(cluster1, 20), new ClusterWeight(cluster2, 80)))))); + verify(mockListener).onResult(resolutionResultCaptor.capture()); + ResolutionResult result = resolutionResultCaptor.getValue(); + assertThat(result.getAddresses()).isEmpty(); + assertServiceConfigForLoadBalancingConfig( + Arrays.asList(cluster1, cluster2), (Map) result.getServiceConfig().getConfig()); + assertThat(result.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL)).isNotNull(); + InternalConfigSelector configSelector = result.getAttributes().get(InternalConfigSelector.KEY); + Result selectResult = configSelector.selectConfig( + new PickSubchannelArgsImpl(call1.methodDescriptor, new Metadata(), CallOptions.DEFAULT)); + assertThat(selectResult.getStatus().isOk()).isTrue(); + assertThat(selectResult.getCallOptions().getOption(XdsNameResolver.CLUSTER_SELECTION_KEY)) + .isEqualTo(cluster2); + assertServiceConfigForMethodConfig(20.0, (Map) selectResult.getConfig()); - String expectedJson4 = "{\n" - + " \"prefix\": \"/\",\n" - + " \"headers\": [\n" - + " {\n" - + " \"name\": \":scheme\",\n" - + " \"exactMatch\": \"https\",\n" - + " \"invertMatch\": false\n" - + " },\n" - + " {\n" - + " \"name\": \":path\",\n" - + " \"regexMatch\": \"google.*\",\n" - + " \"invertMatch\": true\n" - + " },\n" - + " {\n" - + " \"name\": \"timeout\",\n" - + " \"presentMatch\": true,\n" - + " \"invertMatch\": false\n" - + " },\n" - + " {\n" - + " \"name\": \":authority\",\n" - + " \"prefixMatch\": \"google\",\n" - + " \"invertMatch\": false\n" - + " },\n" - + " {\n" - + " \"name\": \":authority\",\n" - + " \"suffixMatch\": \"grpc.io\",\n" - + " \"invertMatch\": false\n" - + " }\n" - + " ],\n" - + " \"action\": \"action_foo\"" - + "}"; - assertThat(XdsNameResolver.convertToRawRoute(routeMatch4, "action_foo")) - .isEqualTo(JsonParser.parse(expectedJson4)); + selectResult = configSelector.selectConfig( + new PickSubchannelArgsImpl(call1.methodDescriptor, new Metadata(), CallOptions.DEFAULT)); + assertThat(selectResult.getStatus().isOk()).isTrue(); + assertThat(selectResult.getCallOptions().getOption(XdsNameResolver.CLUSTER_SELECTION_KEY)) + .isEqualTo(cluster1); + assertServiceConfigForMethodConfig(20.0, (Map) selectResult.getConfig()); } - /** Asserts that the given action contains a single CDS policy with the given cluster name. */ @SuppressWarnings("unchecked") - static void assertCdsPolicy(Map action, String clusterName) { - assertThat(action.keySet()).containsExactly("childPolicy"); - Map lbConfig = - Iterables.getOnlyElement((List>) action.get("childPolicy")); - assertThat(lbConfig.keySet()).containsExactly(CDS_POLICY_NAME); - Map rawConfigValues = (Map) lbConfig.get(CDS_POLICY_NAME); - assertThat(rawConfigValues).containsExactly("cluster", clusterName); + private static Result assertCallSelectResult( + CallInfo call, InternalConfigSelector configSelector, String expectedCluster, + double expectedTimeoutSec) { + Result result = configSelector.selectConfig( + new PickSubchannelArgsImpl(call.methodDescriptor, new Metadata(), CallOptions.DEFAULT)); + assertThat(result.getStatus().isOk()).isTrue(); + assertThat(result.getCallOptions().getOption(XdsNameResolver.CLUSTER_SELECTION_KEY)) + .isEqualTo(expectedCluster); + assertServiceConfigForMethodConfig(expectedTimeoutSec, (Map) result.getConfig()); + return result; + } + + @SuppressWarnings("unchecked") + private InternalConfigSelector resolveToClusters() { + resolver.start(mockListener); + FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); + xdsClient.deliverRoutes( + Arrays.asList( + new Route( + new RouteMatch(null, call1.getFullMethodNameForPath()), + new RouteAction(TimeUnit.SECONDS.toNanos(15L), cluster1, null)), + new Route( + new RouteMatch(null, call2.getFullMethodNameForPath()), + new RouteAction(TimeUnit.SECONDS.toNanos(15L), cluster2, null)))); + verify(mockListener).onResult(resolutionResultCaptor.capture()); + ResolutionResult result = resolutionResultCaptor.getValue(); + assertThat(result.getAddresses()).isEmpty(); + assertServiceConfigForLoadBalancingConfig( + Arrays.asList(cluster1, cluster2), (Map) result.getServiceConfig().getConfig()); + assertThat(result.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL)).isNotNull(); + return result.getAttributes().get(InternalConfigSelector.KEY); } /** - * Asserts that the given action contains a single weighted-target policy with the given cluster - * to weight mapping. + * Verifies the raw service config contains a single method config for method with the + * specified timeout. */ - @SuppressWarnings("unchecked") - static void assertWeightedTargetPolicy( - Map action, Map clusterWeights) { - assertThat(action.keySet()).containsExactly("childPolicy"); - Map lbConfig = - Iterables.getOnlyElement((List>) action.get("childPolicy")); - assertThat(lbConfig.keySet()).containsExactly(WEIGHTED_TARGET_POLICY_NAME); - Map rawConfigValues = (Map) lbConfig.get(WEIGHTED_TARGET_POLICY_NAME); - assertWeightedTargetConfigClusterWeights(rawConfigValues, clusterWeights); + private static void assertServiceConfigForMethodConfig( + double timeoutSec, Map actualServiceConfig) { + List> rawMethodConfigs = + JsonUtil.getListOfObjects(actualServiceConfig, "methodConfig"); + Map methodConfig = Iterables.getOnlyElement(rawMethodConfigs); + List> methods = JsonUtil.getListOfObjects(methodConfig, "name"); + assertThat(Iterables.getOnlyElement(methods)).isEmpty(); + assertThat(JsonUtil.getString(methodConfig, "timeout")).isEqualTo(timeoutSec + "s"); } /** - * Asserts that the given raw config is a weighted-target config with the given cluster to weight - * mapping. + * Verifies the raw service config contains an xDS load balancing config for the given clusters. */ + private static void assertServiceConfigForLoadBalancingConfig( + List clusters, Map actualServiceConfig) { + List> rawLbConfigs = + JsonUtil.getListOfObjects(actualServiceConfig, "loadBalancingConfig"); + Map lbConfig = Iterables.getOnlyElement(rawLbConfigs); + assertThat(lbConfig.keySet()).containsExactly("cluster_manager_experimental"); + Map clusterManagerLbConfig = + JsonUtil.getObject(lbConfig, "cluster_manager_experimental"); + Map clusterManagerChildLbPolicies = + JsonUtil.getObject(clusterManagerLbConfig, "childPolicy"); + assertThat(clusterManagerChildLbPolicies.keySet()).containsExactlyElementsIn(clusters); + for (String cluster : clusters) { + Map childLbConfig = JsonUtil.getObject(clusterManagerChildLbPolicies, cluster); + assertThat(childLbConfig.keySet()).containsExactly("lbPolicy"); + List> childLbConfigValues = + JsonUtil.getListOfObjects(childLbConfig, "lbPolicy"); + Map cdsLbPolicy = Iterables.getOnlyElement(childLbConfigValues); + assertThat(cdsLbPolicy.keySet()).containsExactly("cds_experimental"); + assertThat(JsonUtil.getObject(cdsLbPolicy, "cds_experimental")) + .containsExactly("cluster", cluster); + } + } + @SuppressWarnings("unchecked") - static void assertWeightedTargetConfigClusterWeights( - Map rawConfigValues, Map clusterWeight) { - assertThat(rawConfigValues.keySet()).containsExactly("targets"); - Map targets = (Map) rawConfigValues.get("targets"); - assertThat(targets.keySet()).isEqualTo(clusterWeight.keySet()); - for (String targetName : targets.keySet()) { - Map target = (Map) targets.get(targetName); - assertThat(target.keySet()).containsExactly("childPolicy", "weight"); - Map lbConfig = - Iterables.getOnlyElement((List>) target.get("childPolicy")); - assertThat(lbConfig.keySet()).containsExactly(CDS_POLICY_NAME); - Map rawClusterConfigValues = (Map) lbConfig.get(CDS_POLICY_NAME); - assertThat(rawClusterConfigValues).containsExactly("cluster", targetName); - assertThat(target.get("weight")).isEqualTo(clusterWeight.get(targetName)); + @Test + public void generateServiceConfig_forLoadBalancingConfig() throws IOException { + List clusters = Arrays.asList("cluster-foo", "cluster-bar", "cluster-baz"); + String expectedServiceConfigJson = "{\n" + + " \"loadBalancingConfig\": [{\n" + + " \"cluster_manager_experimental\": {\n" + + " \"childPolicy\": {\n" + + " \"cluster-foo\": {\n" + + " \"lbPolicy\": [{\n" + + " \"cds_experimental\": {\n" + + " \"cluster\": \"cluster-foo\"\n" + + " }\n" + + " }]\n" + + " },\n" + + " \"cluster-bar\": {\n" + + " \"lbPolicy\": [{\n" + + " \"cds_experimental\": {\n" + + " \"cluster\": \"cluster-bar\"\n" + + " }\n" + + " }]\n" + + " },\n" + + " \"cluster-baz\": {\n" + + " \"lbPolicy\": [{\n" + + " \"cds_experimental\": {\n" + + " \"cluster\": \"cluster-baz\"\n" + + " }\n" + + " }]\n" + + " }\n" + + " }\n" + + " }\n" + + " }]\n" + + "}"; + Map expectedServiceConfig = + (Map) JsonParser.parse(expectedServiceConfigJson); + assertThat(XdsNameResolver.generateServiceConfigWithLoadBalancingConfig(clusters)) + .isEqualTo(expectedServiceConfig); + } + + @SuppressWarnings("unchecked") + @Test + public void generateServiceConfig_forMethodTimeoutConfig() throws IOException { + long timeoutNano = TimeUnit.SECONDS.toNanos(1L) + 1L; // 1.0000000001s + String expectedServiceConfigJson = "{\n" + + " \"methodConfig\": [{\n" + + " \"name\": [ {} ],\n" + + " \"timeout\": \"1.000000001s\"\n" + + " }]\n" + + "}"; + Map expectedServiceConfig = + (Map) JsonParser.parse(expectedServiceConfigJson); + assertThat(XdsNameResolver.generateServiceConfigWithMethodTimeoutConfig(timeoutNano)) + .isEqualTo(expectedServiceConfig); + } + + private final class FakeXdsClientPoolFactory implements XdsClientPoolFactory { + @Override + public ObjectPool newXdsClientObjectPool(BootstrapInfo bootstrapInfo) { + return new ObjectPool() { + @Override + public XdsClient getObject() { + return new FakeXdsClient(); + } + + @Override + public XdsClient returnObject(Object object) { + return null; + } + }; + } + } + + private class FakeXdsClient extends XdsClient { + private String resource; + private ConfigWatcher watcher; + + @Override + void watchConfigData(String targetAuthority, ConfigWatcher watcher) { + resource = targetAuthority; + this.watcher = watcher; + } + + @Override + void shutdown() { + // no-op + } + + void deliverRoutes(final List routes) { + syncContext.execute(new Runnable() { + @Override + public void run() { + watcher.onConfigChanged(ConfigUpdate.newBuilder().addRoutes(routes).build()); + } + }); + } + + void deliverError(final Status error) { + syncContext.execute(new Runnable() { + @Override + public void run() { + watcher.onError(error); + } + }); + } + + void deliverResourceNotFound() { + Preconditions.checkState(resource != null, "no resource subscribed"); + syncContext.execute(new Runnable() { + @Override + public void run() { + watcher.onResourceDoesNotExist(resource); + } + }); + } + } + + private static final class CallInfo { + private final String service; + private final String method; + private final MethodDescriptor methodDescriptor; + + private CallInfo(String service, String method) { + this.service = service; + this.method = method; + methodDescriptor = + MethodDescriptor.newBuilder() + .setType(MethodType.UNARY).setFullMethodName(service + "/" + method) + .setRequestMarshaller(TestMethodDescriptors.voidMarshaller()) + .setResponseMarshaller(TestMethodDescriptors.voidMarshaller()) + .build(); + } + + private String getFullMethodNameForPath() { + return "/" + service + "/" + method; } } } From d5dcfa737a991fd90498b7b1fa955ef8c19625b3 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Fri, 11 Sep 2020 19:14:25 -0400 Subject: [PATCH 39/86] all: remove deprecated internal OverrideAuthorityChecker --- .../internal/ManagedChannelImplBuilder.java | 16 --------- .../ManagedChannelImplBuilderTest.java | 27 -------------- .../netty/InternalNettyChannelBuilder.java | 20 ----------- .../io/grpc/netty/NettyChannelBuilder.java | 8 ----- .../grpc/netty/NettyChannelBuilderTest.java | 36 ------------------- .../io/grpc/okhttp/OkHttpChannelBuilder.java | 13 ------- .../grpc/okhttp/OkHttpChannelBuilderTest.java | 11 ------ 7 files changed, 131 deletions(-) diff --git a/core/src/main/java/io/grpc/internal/ManagedChannelImplBuilder.java b/core/src/main/java/io/grpc/internal/ManagedChannelImplBuilder.java index 02af7ef048c..d2807df200b 100644 --- a/core/src/main/java/io/grpc/internal/ManagedChannelImplBuilder.java +++ b/core/src/main/java/io/grpc/internal/ManagedChannelImplBuilder.java @@ -29,9 +29,6 @@ public final class ManagedChannelImplBuilder extends AbstractManagedChannelImplBuilder { private boolean authorityCheckerDisabled; - @Deprecated - @Nullable - private OverrideAuthorityChecker authorityChecker; /** * An interface for Transport implementors to provide the {@link ClientTransportFactory} @@ -137,24 +134,11 @@ public ManagedChannelImplBuilder enableCheckAuthority() { return this; } - @Deprecated - public interface OverrideAuthorityChecker { - String checkAuthority(String authority); - } - - @Deprecated - public void overrideAuthorityChecker(@Nullable OverrideAuthorityChecker authorityChecker) { - this.authorityChecker = authorityChecker; - } - @Override protected String checkAuthority(String authority) { if (authorityCheckerDisabled) { return authority; } - if (authorityChecker != null) { - return authorityChecker.checkAuthority(authority); - } return super.checkAuthority(authority); } diff --git a/core/src/test/java/io/grpc/internal/ManagedChannelImplBuilderTest.java b/core/src/test/java/io/grpc/internal/ManagedChannelImplBuilderTest.java index 19a2f800965..5d3f4cf4f14 100644 --- a/core/src/test/java/io/grpc/internal/ManagedChannelImplBuilderTest.java +++ b/core/src/test/java/io/grpc/internal/ManagedChannelImplBuilderTest.java @@ -132,31 +132,4 @@ public void disableCheckAuthority_invalidAuthorityFailed() { builder.disableCheckAuthority().enableCheckAuthority(); builder.checkAuthority(DUMMY_AUTHORITY_INVALID); } - - /** Ensure authority check can disabled with custom authority check implementation. */ - @Test - @SuppressWarnings("deprecation") - public void overrideAuthorityChecker_default() { - builder.overrideAuthorityChecker( - new io.grpc.internal.ManagedChannelImplBuilder.OverrideAuthorityChecker() { - @Override public String checkAuthority(String authority) { - return authority; - } - }); - assertEquals(DUMMY_AUTHORITY_INVALID, builder.checkAuthority(DUMMY_AUTHORITY_INVALID)); - } - - /** Ensure custom authority is ignored after disableCheckAuthority(). */ - @Test - @SuppressWarnings("deprecation") - public void overrideAuthorityChecker_ignored() { - builder.overrideAuthorityChecker( - new io.grpc.internal.ManagedChannelImplBuilder.OverrideAuthorityChecker() { - @Override public String checkAuthority(String authority) { - throw new IllegalArgumentException(); - } - }); - builder.disableCheckAuthority(); - assertEquals(DUMMY_AUTHORITY_INVALID, builder.checkAuthority(DUMMY_AUTHORITY_INVALID)); - } } diff --git a/netty/src/main/java/io/grpc/netty/InternalNettyChannelBuilder.java b/netty/src/main/java/io/grpc/netty/InternalNettyChannelBuilder.java index 29423389193..ff4d4074ecb 100644 --- a/netty/src/main/java/io/grpc/netty/InternalNettyChannelBuilder.java +++ b/netty/src/main/java/io/grpc/netty/InternalNettyChannelBuilder.java @@ -28,26 +28,6 @@ @Internal public final class InternalNettyChannelBuilder { - /** - * Checks authority upon channel construction. The purpose of this interface is to raise the - * visibility of {@link NettyChannelBuilder.OverrideAuthorityChecker}. - * @deprecated To be removed, use {@link #disableCheckAuthority(NettyChannelBuilder builder)} to - * disable authority check. - */ - @Deprecated - public interface OverrideAuthorityChecker extends NettyChannelBuilder.OverrideAuthorityChecker {} - - /** - * Overrides authority checker. - * @deprecated To be removed, use {@link #disableCheckAuthority(NettyChannelBuilder builder)} to - * disable authority check. - */ - @Deprecated - public static void overrideAuthorityChecker( - NettyChannelBuilder channelBuilder, OverrideAuthorityChecker authorityChecker) { - channelBuilder.overrideAuthorityChecker(authorityChecker); - } - public static void disableCheckAuthority(NettyChannelBuilder builder) { builder.disableCheckAuthority(); } diff --git a/netty/src/main/java/io/grpc/netty/NettyChannelBuilder.java b/netty/src/main/java/io/grpc/netty/NettyChannelBuilder.java index 43d6b96d507..9757cbb3f98 100644 --- a/netty/src/main/java/io/grpc/netty/NettyChannelBuilder.java +++ b/netty/src/main/java/io/grpc/netty/NettyChannelBuilder.java @@ -517,14 +517,6 @@ static ProtocolNegotiator createProtocolNegotiatorByType( } } - @Deprecated - interface OverrideAuthorityChecker extends ManagedChannelImplBuilder.OverrideAuthorityChecker {} - - @Deprecated - void overrideAuthorityChecker(@Nullable OverrideAuthorityChecker authorityChecker) { - this.managedChannelImplBuilder.overrideAuthorityChecker(authorityChecker); - } - NettyChannelBuilder disableCheckAuthority() { this.managedChannelImplBuilder.disableCheckAuthority(); return this; diff --git a/netty/src/test/java/io/grpc/netty/NettyChannelBuilderTest.java b/netty/src/test/java/io/grpc/netty/NettyChannelBuilderTest.java index 9a96d73e5c5..94fc74f5776 100644 --- a/netty/src/test/java/io/grpc/netty/NettyChannelBuilderTest.java +++ b/netty/src/test/java/io/grpc/netty/NettyChannelBuilderTest.java @@ -22,7 +22,6 @@ import static org.mockito.Mockito.mock; import io.grpc.ManagedChannel; -import io.grpc.internal.GrpcUtil; import io.grpc.netty.NettyTestUtil.TrackingObjectPoolForTest; import io.netty.channel.Channel; import io.netty.channel.ChannelFactory; @@ -91,41 +90,6 @@ private void overrideAuthorityIsReadableHelper(NettyChannelBuilder builder, } } - @Test - @Deprecated - public void overrideAllowsInvalidAuthority() { - NettyChannelBuilder builder = new NettyChannelBuilder(new SocketAddress(){}); - InternalNettyChannelBuilder.overrideAuthorityChecker(builder, - new io.grpc.netty.InternalNettyChannelBuilder.OverrideAuthorityChecker() { - @Override - public String checkAuthority(String authority) { - return authority; - } - }); - Object unused = builder.overrideAuthority("[invalidauthority") - .negotiationType(NegotiationType.PLAINTEXT) - .buildTransportFactory(); - } - - @Test - @Deprecated - public void overrideFailsInvalidAuthority() { - NettyChannelBuilder builder = new NettyChannelBuilder(new SocketAddress(){}); - InternalNettyChannelBuilder.overrideAuthorityChecker(builder, - new io.grpc.netty.InternalNettyChannelBuilder.OverrideAuthorityChecker() { - @Override - public String checkAuthority(String authority) { - return GrpcUtil.checkAuthority(authority); - } - }); - - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("Invalid authority:"); - Object unused = builder.overrideAuthority("[invalidauthority") - .negotiationType(NegotiationType.PLAINTEXT) - .buildTransportFactory(); - } - @Test public void failOverrideInvalidAuthority() { NettyChannelBuilder builder = new NettyChannelBuilder(new SocketAddress(){}); diff --git a/okhttp/src/main/java/io/grpc/okhttp/OkHttpChannelBuilder.java b/okhttp/src/main/java/io/grpc/okhttp/OkHttpChannelBuilder.java index 2b2363781de..a7759127e62 100644 --- a/okhttp/src/main/java/io/grpc/okhttp/OkHttpChannelBuilder.java +++ b/okhttp/src/main/java/io/grpc/okhttp/OkHttpChannelBuilder.java @@ -146,7 +146,6 @@ protected OkHttpChannelBuilder(String host, int port) { this(GrpcUtil.authorityFromHostAndPort(host, port)); } - @SuppressWarnings("deprecation") private OkHttpChannelBuilder(String target) { final class OkHttpChannelTransportFactoryBuilder implements ClientTransportFactoryBuilder { @Override @@ -165,14 +164,6 @@ public int getDefaultPort() { managedChannelImplBuilder = new ManagedChannelImplBuilder(target, new OkHttpChannelTransportFactoryBuilder(), new OkHttpChannelDefaultPortProvider()); - - managedChannelImplBuilder.overrideAuthorityChecker( - new io.grpc.internal.ManagedChannelImplBuilder.OverrideAuthorityChecker() { - @Override - public String checkAuthority(String authority) { - return OkHttpChannelBuilder.this.checkAuthority(authority); - } - }); } @Internal @@ -433,10 +424,6 @@ final ClientTransportFactory buildTransportFactory() { useGetForSafeMethods); } - protected String checkAuthority(String authority) { - return GrpcUtil.checkAuthority(authority); - } - final OkHttpChannelBuilder disableCheckAuthority() { this.managedChannelImplBuilder.disableCheckAuthority(); return this; diff --git a/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java b/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java index 5be8e95ff80..4aa1d1aa53c 100644 --- a/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java +++ b/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java @@ -80,17 +80,6 @@ public void failOverrideInvalidAuthority() { builder.overrideAuthority("[invalidauthority"); } - @Test - public void checkAuthorityOverrideAllowsInvalidAuthority() { - OkHttpChannelBuilder builder = new OkHttpChannelBuilder("good", 1234) { - @Override - protected String checkAuthority(String authority) { - return authority; - } - }; - builder.overrideAuthority("[invalidauthority").usePlaintext().buildTransportFactory(); - } - @Test public void disableCheckAuthorityAllowsInvalidAuthority() { OkHttpChannelBuilder builder = new OkHttpChannelBuilder("good", 1234) From af6fbf6b7448768d214f8df1888eb7064bd3c650 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Fri, 11 Sep 2020 18:42:00 -0400 Subject: [PATCH 40/86] okhttp: make OkHttpChannelBuilder final --- .../io/grpc/okhttp/OkHttpChannelBuilder.java | 39 +++++++++---------- .../grpc/okhttp/OkHttpChannelBuilderTest.java | 6 +-- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/okhttp/src/main/java/io/grpc/okhttp/OkHttpChannelBuilder.java b/okhttp/src/main/java/io/grpc/okhttp/OkHttpChannelBuilder.java index a7759127e62..93ea57e2df4 100644 --- a/okhttp/src/main/java/io/grpc/okhttp/OkHttpChannelBuilder.java +++ b/okhttp/src/main/java/io/grpc/okhttp/OkHttpChannelBuilder.java @@ -58,7 +58,7 @@ /** Convenience class for building channels with the OkHttp transport. */ @ExperimentalApi("https://github.com/grpc/grpc-java/issues/1785") -public class OkHttpChannelBuilder extends ForwardingChannelBuilder { +public final class OkHttpChannelBuilder extends ForwardingChannelBuilder { public static final int DEFAULT_FLOW_CONTROL_WINDOW = 65535; private final ManagedChannelImplBuilder managedChannelImplBuilder; @@ -142,7 +142,7 @@ public static OkHttpChannelBuilder forTarget(String target) { */ private final boolean useGetForSafeMethods = false; - protected OkHttpChannelBuilder(String host, int port) { + private OkHttpChannelBuilder(String host, int port) { this(GrpcUtil.authorityFromHostAndPort(host, port)); } @@ -168,13 +168,12 @@ public int getDefaultPort() { @Internal @Override - protected final ManagedChannelBuilder delegate() { + protected ManagedChannelBuilder delegate() { return managedChannelImplBuilder; } @VisibleForTesting - final OkHttpChannelBuilder setTransportTracerFactory( - TransportTracer.Factory transportTracerFactory) { + OkHttpChannelBuilder setTransportTracerFactory(TransportTracer.Factory transportTracerFactory) { this.transportTracerFactory = transportTracerFactory; return this; } @@ -185,7 +184,7 @@ final OkHttpChannelBuilder setTransportTracerFactory( *

The channel does not take ownership of the given executor. It is the caller' responsibility * to shutdown the executor when appropriate. */ - public final OkHttpChannelBuilder transportExecutor(@Nullable Executor transportExecutor) { + public OkHttpChannelBuilder transportExecutor(@Nullable Executor transportExecutor) { this.transportExecutor = transportExecutor; return this; } @@ -196,7 +195,7 @@ public final OkHttpChannelBuilder transportExecutor(@Nullable Executor transport * * @since 1.20.0 */ - public final OkHttpChannelBuilder socketFactory(@Nullable SocketFactory socketFactory) { + public OkHttpChannelBuilder socketFactory(@Nullable SocketFactory socketFactory) { this.socketFactory = socketFactory; return this; } @@ -214,7 +213,7 @@ public final OkHttpChannelBuilder socketFactory(@Nullable SocketFactory socketFa * @deprecated use {@link #usePlaintext()} or {@link #useTransportSecurity()} instead. */ @Deprecated - public final OkHttpChannelBuilder negotiationType(io.grpc.okhttp.NegotiationType type) { + public OkHttpChannelBuilder negotiationType(io.grpc.okhttp.NegotiationType type) { Preconditions.checkNotNull(type, "type"); switch (type) { case TLS: @@ -284,7 +283,7 @@ public OkHttpChannelBuilder keepAliveWithoutCalls(boolean enable) { /** * Override the default {@link SSLSocketFactory} and enable TLS negotiation. */ - public final OkHttpChannelBuilder sslSocketFactory(SSLSocketFactory factory) { + public OkHttpChannelBuilder sslSocketFactory(SSLSocketFactory factory) { this.sslSocketFactory = factory; negotiationType = NegotiationType.TLS; return this; @@ -310,7 +309,7 @@ public final OkHttpChannelBuilder sslSocketFactory(SSLSocketFactory factory) { * @return this * */ - public final OkHttpChannelBuilder hostnameVerifier(@Nullable HostnameVerifier hostnameVerifier) { + public OkHttpChannelBuilder hostnameVerifier(@Nullable HostnameVerifier hostnameVerifier) { this.hostnameVerifier = hostnameVerifier; return this; } @@ -327,7 +326,7 @@ public final OkHttpChannelBuilder hostnameVerifier(@Nullable HostnameVerifier ho * @throws IllegalArgumentException * If {@code connectionSpec} is not with TLS */ - public final OkHttpChannelBuilder connectionSpec( + public OkHttpChannelBuilder connectionSpec( com.squareup.okhttp.ConnectionSpec connectionSpec) { Preconditions.checkArgument(connectionSpec.isTls(), "plaintext ConnectionSpec is not accepted"); this.connectionSpec = Utils.convertSpec(connectionSpec); @@ -336,7 +335,7 @@ public final OkHttpChannelBuilder connectionSpec( /** Sets the negotiation type for the HTTP/2 connection to plaintext. */ @Override - public final OkHttpChannelBuilder usePlaintext() { + public OkHttpChannelBuilder usePlaintext() { negotiationType = NegotiationType.PLAINTEXT; return this; } @@ -350,7 +349,7 @@ public final OkHttpChannelBuilder usePlaintext() { * socket factory used. */ @Override - public final OkHttpChannelBuilder useTransportSecurity() { + public OkHttpChannelBuilder useTransportSecurity() { negotiationType = NegotiationType.TLS; return this; } @@ -365,7 +364,7 @@ public final OkHttpChannelBuilder useTransportSecurity() { * * @since 1.11.0 */ - public final OkHttpChannelBuilder scheduledExecutorService( + public OkHttpChannelBuilder scheduledExecutorService( ScheduledExecutorService scheduledExecutorService) { this.scheduledExecutorService = checkNotNull(scheduledExecutorService, "scheduledExecutorService"); @@ -398,13 +397,13 @@ public OkHttpChannelBuilder maxInboundMetadataSize(int bytes) { * RESOURCE_EXHAUSTED. */ @Override - public final OkHttpChannelBuilder maxInboundMessageSize(int max) { + public OkHttpChannelBuilder maxInboundMessageSize(int max) { Preconditions.checkArgument(max >= 0, "negative max"); maxInboundMessageSize = max; return this; } - final ClientTransportFactory buildTransportFactory() { + ClientTransportFactory buildTransportFactory() { boolean enableKeepAlive = keepAliveTimeNanos != KEEPALIVE_TIME_NANOS_DISABLED; return new OkHttpTransportFactory( transportExecutor, @@ -424,17 +423,17 @@ final ClientTransportFactory buildTransportFactory() { useGetForSafeMethods); } - final OkHttpChannelBuilder disableCheckAuthority() { + OkHttpChannelBuilder disableCheckAuthority() { this.managedChannelImplBuilder.disableCheckAuthority(); return this; } - final OkHttpChannelBuilder enableCheckAuthority() { + OkHttpChannelBuilder enableCheckAuthority() { this.managedChannelImplBuilder.enableCheckAuthority(); return this; } - final int getDefaultPort() { + int getDefaultPort() { switch (negotiationType) { case PLAINTEXT: return GrpcUtil.DEFAULT_PORT_PLAINTEXT; @@ -445,7 +444,7 @@ final int getDefaultPort() { } } - final void setStatsEnabled(boolean value) { + void setStatsEnabled(boolean value) { this.managedChannelImplBuilder.setStatsEnabled(value); } diff --git a/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java b/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java index 4aa1d1aa53c..60ec5da1c4e 100644 --- a/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java +++ b/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java @@ -73,7 +73,7 @@ private void overrideAuthorityIsReadableHelper(OkHttpChannelBuilder builder, @Test public void failOverrideInvalidAuthority() { - OkHttpChannelBuilder builder = new OkHttpChannelBuilder("good", 1234); + OkHttpChannelBuilder builder = OkHttpChannelBuilder.forAddress("good", 1234); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Invalid authority:"); @@ -82,14 +82,14 @@ public void failOverrideInvalidAuthority() { @Test public void disableCheckAuthorityAllowsInvalidAuthority() { - OkHttpChannelBuilder builder = new OkHttpChannelBuilder("good", 1234) + OkHttpChannelBuilder builder = OkHttpChannelBuilder.forAddress("good", 1234) .disableCheckAuthority(); builder.overrideAuthority("[invalidauthority").usePlaintext().buildTransportFactory(); } @Test public void enableCheckAuthorityFailOverrideInvalidAuthority() { - OkHttpChannelBuilder builder = new OkHttpChannelBuilder("good", 1234) + OkHttpChannelBuilder builder = OkHttpChannelBuilder.forAddress("good", 1234) .disableCheckAuthority() .enableCheckAuthority(); From 9dd56a7f0f57b59333e0383e8f5f34b717b054b3 Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Mon, 14 Sep 2020 16:26:28 -0700 Subject: [PATCH 41/86] xds: throw XdsInitializationException when reading bootstrap file encounters error (#7420) Introduce XdsInitializationException, which is thrown when gRPC fails to read the xDS bootstrap information, or fails to create the XdsClient object with loaded bootstrap configurations. gRPC components (e.g., the XdsNameResolver) is expected to propagate such exceptions gracefully to the channel. --- .../main/java/io/grpc/xds/Bootstrapper.java | 43 ++++++++++++------- .../xds/XdsClientWrapperForServerSds.java | 18 ++------ .../grpc/xds/XdsInitializationException.java | 32 ++++++++++++++ .../sds/ClientSslContextProviderFactory.java | 10 ++--- .../sds/ServerSslContextProviderFactory.java | 10 ++--- .../java/io/grpc/xds/BootstrapperTest.java | 39 ++++++++--------- .../XdsClientWrapperForServerSdsTestMisc.java | 2 +- .../java/io/grpc/xds/XdsNameResolverTest.java | 4 +- .../io/grpc/xds/XdsServerBuilderTest.java | 2 +- .../CommonCertProviderTestUtils.java | 25 +++++++---- ...MeshCaCertificateProviderProviderTest.java | 38 ++++++++++------ .../ClientSslContextProviderFactoryTest.java | 20 ++++++--- .../ServerSslContextProviderFactoryTest.java | 20 ++++++--- 13 files changed, 162 insertions(+), 101 deletions(-) create mode 100644 xds/src/main/java/io/grpc/xds/XdsInitializationException.java diff --git a/xds/src/main/java/io/grpc/xds/Bootstrapper.java b/xds/src/main/java/io/grpc/xds/Bootstrapper.java index ea59498a272..19856b88a66 100644 --- a/xds/src/main/java/io/grpc/xds/Bootstrapper.java +++ b/xds/src/main/java/io/grpc/xds/Bootstrapper.java @@ -53,17 +53,22 @@ public abstract class Bootstrapper { private static final Bootstrapper DEFAULT_INSTANCE = new Bootstrapper() { @Override - public BootstrapInfo readBootstrap() throws IOException { + public BootstrapInfo readBootstrap() throws XdsInitializationException { String filePath = System.getenv(BOOTSTRAP_PATH_SYS_ENV_VAR); if (filePath == null) { - throw - new IOException("Environment variable " + BOOTSTRAP_PATH_SYS_ENV_VAR + " not defined."); + throw new XdsInitializationException( + "Environment variable " + BOOTSTRAP_PATH_SYS_ENV_VAR + " not defined."); } XdsLogger .withPrefix(LOG_PREFIX) .log(XdsLogLevel.INFO, BOOTSTRAP_PATH_SYS_ENV_VAR + "={0}", filePath); - return parseConfig( - new String(Files.readAllBytes(Paths.get(filePath)), StandardCharsets.UTF_8)); + String fileContent; + try { + fileContent = new String(Files.readAllBytes(Paths.get(filePath)), StandardCharsets.UTF_8); + } catch (IOException e) { + throw new XdsInitializationException("Fail to read bootstrap file", e); + } + return parseConfig(fileContent); } }; @@ -74,42 +79,47 @@ public static Bootstrapper getInstance() { /** * Returns configurations from bootstrap. */ - public abstract BootstrapInfo readBootstrap() throws IOException; + public abstract BootstrapInfo readBootstrap() throws XdsInitializationException; /** Parses a raw string into {@link BootstrapInfo}. */ @VisibleForTesting - @SuppressWarnings("deprecation") - public static BootstrapInfo parseConfig(String rawData) throws IOException { + @SuppressWarnings("unchecked") + public static BootstrapInfo parseConfig(String rawData) throws XdsInitializationException { XdsLogger logger = XdsLogger.withPrefix(LOG_PREFIX); logger.log(XdsLogLevel.INFO, "Reading bootstrap information"); - @SuppressWarnings("unchecked") - Map rawBootstrap = (Map) JsonParser.parse(rawData); + Map rawBootstrap; + try { + rawBootstrap = (Map) JsonParser.parse(rawData); + } catch (IOException e) { + throw new XdsInitializationException("Failed to parse JSON", e); + } logger.log(XdsLogLevel.DEBUG, "Bootstrap configuration:\n{0}", rawBootstrap); List servers = new ArrayList<>(); List rawServerConfigs = JsonUtil.getList(rawBootstrap, "xds_servers"); if (rawServerConfigs == null) { - throw new IOException("Invalid bootstrap: 'xds_servers' does not exist."); + throw new XdsInitializationException("Invalid bootstrap: 'xds_servers' does not exist."); } logger.log(XdsLogLevel.INFO, "Configured with {0} xDS servers", rawServerConfigs.size()); List> serverConfigList = JsonUtil.checkObjectList(rawServerConfigs); for (Map serverConfig : serverConfigList) { String serverUri = JsonUtil.getString(serverConfig, "server_uri"); if (serverUri == null) { - throw new IOException("Invalid bootstrap: 'xds_servers' contains unknown server."); + throw new XdsInitializationException( + "Invalid bootstrap: missing 'xds_servers'"); } logger.log(XdsLogLevel.INFO, "xDS server URI: {0}", serverUri); List channelCredsOptions = new ArrayList<>(); List rawChannelCredsList = JsonUtil.getList(serverConfig, "channel_creds"); if (rawChannelCredsList == null || rawChannelCredsList.isEmpty()) { - throw new IOException( + throw new XdsInitializationException( "Invalid bootstrap: server " + serverUri + " 'channel_creds' required"); } List> channelCredsList = JsonUtil.checkObjectList(rawChannelCredsList); for (Map channelCreds : channelCredsList) { String type = JsonUtil.getString(channelCreds, "type"); if (type == null) { - throw new IOException( + throw new XdsInitializationException( "Invalid bootstrap: server " + serverUri + " with 'channel_creds' type unspecified"); } logger.log(XdsLogLevel.INFO, "Channel credentials option: {0}", type); @@ -182,9 +192,10 @@ public static BootstrapInfo parseConfig(String rawData) throws IOException { return new BootstrapInfo(servers, nodeBuilder.build(), certProviders); } - static T checkForNull(T value, String fieldName) throws IOException { + static T checkForNull(T value, String fieldName) throws XdsInitializationException { if (value == null) { - throw new IOException("Invalid bootstrap: '" + fieldName + "' does not exist."); + throw new XdsInitializationException( + "Invalid bootstrap: '" + fieldName + "' does not exist."); } return value; } diff --git a/xds/src/main/java/io/grpc/xds/XdsClientWrapperForServerSds.java b/xds/src/main/java/io/grpc/xds/XdsClientWrapperForServerSds.java index 995b1051219..e79726e8a1b 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientWrapperForServerSds.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientWrapperForServerSds.java @@ -75,18 +75,6 @@ public final class XdsClientWrapperForServerSds { private XdsClient.ListenerWatcher listenerWatcher; @VisibleForTesting final Set serverWatchers = new HashSet<>(); - /** - * Thrown when no suitable management server was found in the bootstrap file. - */ - public static final class ManagementServerNotFoundException extends IOException { - - private static final long serialVersionUID = 1; - - public ManagementServerNotFoundException(String msg) { - super(msg); - } - } - /** * Creates a {@link XdsClientWrapperForServerSds}. * @@ -137,11 +125,11 @@ public void createXdsClientAndStart() throws IOException { bootstrapInfo = Bootstrapper.getInstance().readBootstrap(); serverList = bootstrapInfo.getServers(); if (serverList.isEmpty()) { - throw new ManagementServerNotFoundException("No management server provided by bootstrap"); + throw new XdsInitializationException("No management server provided by bootstrap"); } - } catch (IOException e) { + } catch (XdsInitializationException e) { reportError(Status.fromThrowable(e)); - throw e; + throw new IOException(e); } Node node = bootstrapInfo.getNode(); timeService = SharedResourceHolder.get(timeServiceResource); diff --git a/xds/src/main/java/io/grpc/xds/XdsInitializationException.java b/xds/src/main/java/io/grpc/xds/XdsInitializationException.java new file mode 100644 index 00000000000..6935840aedb --- /dev/null +++ b/xds/src/main/java/io/grpc/xds/XdsInitializationException.java @@ -0,0 +1,32 @@ +/* + * Copyright 2020 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.xds; + +/** + * Throws when fail to bootstrap or initialize the XdsClient. + */ +public final class XdsInitializationException extends Exception { + private static final long serialVersionUID = 1L; + + public XdsInitializationException(String message) { + super(message); + } + + public XdsInitializationException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/xds/src/main/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactory.java b/xds/src/main/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactory.java index 8e593fd560e..1163e94ab28 100644 --- a/xds/src/main/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactory.java +++ b/xds/src/main/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactory.java @@ -21,9 +21,9 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.grpc.xds.Bootstrapper; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; +import io.grpc.xds.XdsInitializationException; import io.grpc.xds.internal.certprovider.CertProviderClientSslContextProvider; import io.grpc.xds.internal.sds.ReferenceCountingMap.ValueFactory; -import java.io.IOException; import java.util.concurrent.Executors; /** Factory to create client-side SslContextProvider from UpstreamTlsContext. */ @@ -64,8 +64,8 @@ public SslContextProvider create(UpstreamTlsContext upstreamTlsContext) { .setDaemon(true) .build()), /* channelExecutor= */ null); - } catch (IOException ioe) { - throw new RuntimeException(ioe); + } catch (XdsInitializationException e) { + throw new RuntimeException(e); } } else if (CommonTlsContextUtil.hasCertProviderInstance( upstreamTlsContext.getCommonTlsContext())) { @@ -74,8 +74,8 @@ public SslContextProvider create(UpstreamTlsContext upstreamTlsContext) { upstreamTlsContext, bootstrapper.readBootstrap().getNode().toEnvoyProtoNode(), bootstrapper.readBootstrap().getCertProviders()); - } catch (IOException ioe) { - throw new RuntimeException(ioe); + } catch (XdsInitializationException e) { + throw new RuntimeException(e); } } throw new UnsupportedOperationException("Unsupported configurations in UpstreamTlsContext!"); diff --git a/xds/src/main/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactory.java b/xds/src/main/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactory.java index 6ae5308930e..e3f006acee6 100644 --- a/xds/src/main/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactory.java +++ b/xds/src/main/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactory.java @@ -21,9 +21,9 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.grpc.xds.Bootstrapper; import io.grpc.xds.EnvoyServerProtoData.DownstreamTlsContext; +import io.grpc.xds.XdsInitializationException; import io.grpc.xds.internal.certprovider.CertProviderServerSslContextProvider; import io.grpc.xds.internal.sds.ReferenceCountingMap.ValueFactory; -import java.io.IOException; import java.util.concurrent.Executors; /** Factory to create server-side SslContextProvider from DownstreamTlsContext. */ @@ -66,8 +66,8 @@ public SslContextProvider create( .setDaemon(true) .build()), /* channelExecutor= */ null); - } catch (IOException ioe) { - throw new RuntimeException(ioe); + } catch (XdsInitializationException e) { + throw new RuntimeException(e); } } else if (CommonTlsContextUtil.hasCertProviderInstance( downstreamTlsContext.getCommonTlsContext())) { @@ -76,8 +76,8 @@ public SslContextProvider create( downstreamTlsContext, bootstrapper.readBootstrap().getNode().toEnvoyProtoNode(), bootstrapper.readBootstrap().getCertProviders()); - } catch (IOException ioe) { - throw new RuntimeException(ioe); + } catch (XdsInitializationException e) { + throw new RuntimeException(e); } } throw new UnsupportedOperationException("Unsupported configurations in DownstreamTlsContext!"); diff --git a/xds/src/test/java/io/grpc/xds/BootstrapperTest.java b/xds/src/test/java/io/grpc/xds/BootstrapperTest.java index 75b250f4f04..c49e3fe964b 100644 --- a/xds/src/test/java/io/grpc/xds/BootstrapperTest.java +++ b/xds/src/test/java/io/grpc/xds/BootstrapperTest.java @@ -28,7 +28,6 @@ import io.grpc.xds.Bootstrapper.ServerInfo; import io.grpc.xds.EnvoyProtoData.Locality; import io.grpc.xds.EnvoyProtoData.Node; -import java.io.IOException; import java.util.List; import java.util.Map; import org.junit.Rule; @@ -44,7 +43,7 @@ public class BootstrapperTest { @Rule public ExpectedException thrown = ExpectedException.none(); @Test - public void parseBootstrap_validData_singleXdsServer() throws IOException { + public void parseBootstrap_validData_singleXdsServer() throws XdsInitializationException { String rawData = "{\n" + " \"node\": {\n" + " \"id\": \"ENVOY_NODE_ID\",\n" @@ -95,7 +94,7 @@ public void parseBootstrap_validData_singleXdsServer() throws IOException { } @Test - public void parseBootstrap_validData_multipleXdsServers() throws IOException { + public void parseBootstrap_validData_multipleXdsServers() throws XdsInitializationException { String rawData = "{\n" + " \"node\": {\n" + " \"id\": \"ENVOY_NODE_ID\",\n" @@ -163,7 +162,7 @@ public void parseBootstrap_validData_multipleXdsServers() throws IOException { } @Test - public void parseBootstrap_IgnoreIrrelevantFields() throws IOException { + public void parseBootstrap_IgnoreIrrelevantFields() throws XdsInitializationException { String rawData = "{\n" + " \"node\": {\n" + " \"id\": \"ENVOY_NODE_ID\",\n" @@ -216,15 +215,15 @@ public void parseBootstrap_IgnoreIrrelevantFields() throws IOException { } @Test - public void parseBootstrap_emptyData() throws IOException { + public void parseBootstrap_emptyData() throws XdsInitializationException { String rawData = ""; - thrown.expect(IOException.class); + thrown.expect(XdsInitializationException.class); Bootstrapper.parseConfig(rawData); } @Test - public void parseBootstrap_minimumRequiredFields() throws IOException { + public void parseBootstrap_minimumRequiredFields() throws XdsInitializationException { String rawData = "{\n" + " \"xds_servers\": []\n" + "}"; @@ -235,7 +234,7 @@ public void parseBootstrap_minimumRequiredFields() throws IOException { } @Test - public void parseBootstrap_minimalUsableData() throws IOException { + public void parseBootstrap_minimalUsableData() throws XdsInitializationException { String rawData = "{\n" + " \"xds_servers\": [\n" + " {\n" @@ -259,7 +258,7 @@ public void parseBootstrap_minimalUsableData() throws IOException { } @Test - public void parseBootstrap_noXdsServers() throws IOException { + public void parseBootstrap_noXdsServers() throws XdsInitializationException { String rawData = "{\n" + " \"node\": {\n" + " \"id\": \"ENVOY_NODE_ID\",\n" @@ -276,13 +275,13 @@ public void parseBootstrap_noXdsServers() throws IOException { + " }\n" + "}"; - thrown.expect(IOException.class); + thrown.expect(XdsInitializationException.class); thrown.expectMessage("Invalid bootstrap: 'xds_servers' does not exist."); Bootstrapper.parseConfig(rawData); } @Test - public void parseBootstrap_serverWithoutServerUri() throws IOException { + public void parseBootstrap_serverWithoutServerUri() throws XdsInitializationException { String rawData = "{" + " \"node\": {\n" + " \"id\": \"ENVOY_NODE_ID\",\n" @@ -306,13 +305,13 @@ public void parseBootstrap_serverWithoutServerUri() throws IOException { + " ]\n " + "}"; - thrown.expect(IOException.class); - thrown.expectMessage("Invalid bootstrap: 'xds_servers' contains unknown server."); + thrown.expect(XdsInitializationException.class); + thrown.expectMessage("Invalid bootstrap: missing 'xds_servers'"); Bootstrapper.parseConfig(rawData); } @Test - public void parseBootstrap_validData_certProviderInstances() throws IOException { + public void parseBootstrap_validData_certProviderInstances() throws XdsInitializationException { String rawData = "{\n" + " \"xds_servers\": [],\n" @@ -366,7 +365,7 @@ public void parseBootstrap_validData_certProviderInstances() throws IOException } @Test - public void parseBootstrap_badPluginName() throws IOException { + public void parseBootstrap_badPluginName() throws XdsInitializationException { String rawData = "{\n" + " \"xds_servers\": [],\n" @@ -413,7 +412,7 @@ public void parseBootstrap_badPluginName() throws IOException { } @Test - public void parseBootstrap_badConfig() throws IOException { + public void parseBootstrap_badConfig() throws XdsInitializationException { String rawData = "{\n" + " \"xds_servers\": [],\n" @@ -438,7 +437,7 @@ public void parseBootstrap_badConfig() throws IOException { } @Test - public void parseBootstrap_missingConfig() throws IOException { + public void parseBootstrap_missingConfig() { String rawData = "{\n" + " \"xds_servers\": [],\n" @@ -456,7 +455,7 @@ public void parseBootstrap_missingConfig() throws IOException { try { Bootstrapper.parseConfig(rawData); fail("exception expected"); - } catch (IOException expected) { + } catch (XdsInitializationException expected) { assertThat(expected) .hasMessageThat() .isEqualTo("Invalid bootstrap: 'config' does not exist."); @@ -464,7 +463,7 @@ public void parseBootstrap_missingConfig() throws IOException { } @Test - public void parseBootstrap_missingPluginName() throws IOException { + public void parseBootstrap_missingPluginName() { String rawData = "{\n" + " \"xds_servers\": [],\n" @@ -504,7 +503,7 @@ public void parseBootstrap_missingPluginName() throws IOException { try { Bootstrapper.parseConfig(rawData); fail("exception expected"); - } catch (IOException expected) { + } catch (XdsInitializationException expected) { assertThat(expected) .hasMessageThat() .isEqualTo("Invalid bootstrap: 'plugin_name' does not exist."); diff --git a/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTestMisc.java b/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTestMisc.java index d0e34b3fe7d..5fe70e78ff4 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTestMisc.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTestMisc.java @@ -207,7 +207,7 @@ public void startXdsClient_expectException() { verify(mockServerWatcher).onError(argCaptor.capture()); Status captured = argCaptor.getValue(); assertThat(captured.getCode()).isEqualTo(Status.Code.UNKNOWN); - assertThat(captured.getCause()).isInstanceOf(IOException.class); + assertThat(captured.getCause()).isInstanceOf(XdsInitializationException.class); assertThat(captured.getCause()) .hasMessageThat() .contains("Environment variable GRPC_XDS_BOOTSTRAP not defined"); diff --git a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java index b1142380a6f..30658a045fe 100644 --- a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java @@ -126,8 +126,8 @@ public BootstrapInfo readBootstrap() { public void resolve_failToBootstrap() { Bootstrapper bootstrapper = new Bootstrapper() { @Override - public BootstrapInfo readBootstrap() throws IOException { - throw new IOException("Fail to read bootstrap file"); + public BootstrapInfo readBootstrap() throws XdsInitializationException { + throw new XdsInitializationException("Fail to read bootstrap file"); } }; resolver = new XdsNameResolver(AUTHORITY, serviceConfigParser, syncContext, bootstrapper, diff --git a/xds/src/test/java/io/grpc/xds/XdsServerBuilderTest.java b/xds/src/test/java/io/grpc/xds/XdsServerBuilderTest.java index bab78b22647..51e4e782fca 100644 --- a/xds/src/test/java/io/grpc/xds/XdsServerBuilderTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsServerBuilderTest.java @@ -242,7 +242,7 @@ public void xdsServerWithoutMockXdsClient_startError() verify(mockErrorNotifier).onError(argCaptor.capture()); Status captured = argCaptor.getValue(); assertThat(captured.getCode()).isEqualTo(Status.Code.UNKNOWN); - assertThat(captured.getCause()).isInstanceOf(IOException.class); + assertThat(captured.getCause()).isInstanceOf(XdsInitializationException.class); assertThat(captured.getCause()) .hasMessageThat() .contains("Environment variable GRPC_XDS_BOOTSTRAP not defined"); diff --git a/xds/src/test/java/io/grpc/xds/internal/certprovider/CommonCertProviderTestUtils.java b/xds/src/test/java/io/grpc/xds/internal/certprovider/CommonCertProviderTestUtils.java index 4ca3bc1b138..4e70164fe60 100644 --- a/xds/src/test/java/io/grpc/xds/internal/certprovider/CommonCertProviderTestUtils.java +++ b/xds/src/test/java/io/grpc/xds/internal/certprovider/CommonCertProviderTestUtils.java @@ -21,6 +21,7 @@ import com.google.common.io.CharStreams; import io.grpc.internal.testing.TestUtils; import io.grpc.xds.Bootstrapper; +import io.grpc.xds.XdsInitializationException; import io.grpc.xds.internal.sds.trust.CertificateUtils; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; @@ -55,7 +56,8 @@ public class CommonCertProviderTestUtils { Pattern.CASE_INSENSITIVE); /** Creates a test bootstrap info object. */ - public static Bootstrapper.BootstrapInfo getTestBootstrapInfo() throws IOException { + public static Bootstrapper.BootstrapInfo getTestBootstrapInfo() + throws XdsInitializationException { String rawData = "{\n" + " \"xds_servers\": [],\n" @@ -95,7 +97,8 @@ public static Bootstrapper.BootstrapInfo getTestBootstrapInfo() throws IOExcepti return Bootstrapper.parseConfig(rawData); } - static Bootstrapper.BootstrapInfo getNonDefaultTestBootstrapInfo() throws IOException { + static Bootstrapper.BootstrapInfo getNonDefaultTestBootstrapInfo() + throws XdsInitializationException { String rawData = "{\n" + " \"xds_servers\": [],\n" @@ -135,7 +138,8 @@ static Bootstrapper.BootstrapInfo getNonDefaultTestBootstrapInfo() throws IOExce return Bootstrapper.parseConfig(rawData); } - static Bootstrapper.BootstrapInfo getMinimalBootstrapInfo() throws IOException { + static Bootstrapper.BootstrapInfo getMinimalBootstrapInfo() + throws XdsInitializationException { String rawData = "{\n" + " \"xds_servers\": [],\n" @@ -163,7 +167,8 @@ static Bootstrapper.BootstrapInfo getMinimalBootstrapInfo() throws IOException { return Bootstrapper.parseConfig(rawData); } - static Bootstrapper.BootstrapInfo getMinimalBootstrapInfo_v1beta1AndZone() throws IOException { + static Bootstrapper.BootstrapInfo getMinimalBootstrapInfo_v1beta1AndZone() + throws XdsInitializationException { String rawData = "{\n" + " \"xds_servers\": [],\n" @@ -191,7 +196,8 @@ static Bootstrapper.BootstrapInfo getMinimalBootstrapInfo_v1beta1AndZone() throw return Bootstrapper.parseConfig(rawData); } - static Bootstrapper.BootstrapInfo getMinimalAndBadClusterUrlBootstrapInfo() throws IOException { + static Bootstrapper.BootstrapInfo getMinimalAndBadClusterUrlBootstrapInfo() + throws XdsInitializationException { String rawData = "{\n" + " \"xds_servers\": [],\n" @@ -219,7 +225,8 @@ static Bootstrapper.BootstrapInfo getMinimalAndBadClusterUrlBootstrapInfo() thro return Bootstrapper.parseConfig(rawData); } - static Bootstrapper.BootstrapInfo getMissingSaJwtLocation() throws IOException { + static Bootstrapper.BootstrapInfo getMissingSaJwtLocation() + throws XdsInitializationException { String rawData = "{\n" + " \"xds_servers\": [],\n" @@ -246,7 +253,8 @@ static Bootstrapper.BootstrapInfo getMissingSaJwtLocation() throws IOException { return Bootstrapper.parseConfig(rawData); } - static Bootstrapper.BootstrapInfo getMissingGkeClusterUrl() throws IOException { + static Bootstrapper.BootstrapInfo getMissingGkeClusterUrl() + throws XdsInitializationException { String rawData = "{\n" + " \"xds_servers\": [],\n" @@ -285,7 +293,8 @@ static Bootstrapper.BootstrapInfo getMissingGkeClusterUrl() throws IOException { return Bootstrapper.parseConfig(rawData); } - static Bootstrapper.BootstrapInfo getBadChannelCredsConfig() throws IOException { + static Bootstrapper.BootstrapInfo getBadChannelCredsConfig() + throws XdsInitializationException { String rawData = "{\n" + " \"xds_servers\": [],\n" diff --git a/xds/src/test/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderProviderTest.java b/xds/src/test/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderProviderTest.java index 0f92d19cff8..d1bd116703b 100644 --- a/xds/src/test/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderProviderTest.java @@ -31,6 +31,7 @@ import io.grpc.internal.ExponentialBackoffPolicy; import io.grpc.internal.TimeProvider; import io.grpc.xds.Bootstrapper; +import io.grpc.xds.XdsInitializationException; import io.grpc.xds.internal.sts.StsCredentials; import java.io.IOException; import java.util.Map; @@ -106,7 +107,7 @@ public void providerRegisteredName() { } @Test - public void createProvider_minimalConfig() throws IOException { + public void createProvider_minimalConfig() throws XdsInitializationException { CertificateProvider.DistributorWatcher distWatcher = new CertificateProvider.DistributorWatcher(); Map map = buildMinimalConfig(); @@ -141,7 +142,8 @@ public void createProvider_minimalConfig() throws IOException { } @Test - public void createProvider_minimalConfig_v1beta1AndZone() throws IOException { + public void createProvider_minimalConfig_v1beta1AndZone() + throws XdsInitializationException { CertificateProvider.DistributorWatcher distWatcher = new CertificateProvider.DistributorWatcher(); Map map = buildMinimalConfig_v1beta1AndZone(); @@ -176,7 +178,8 @@ public void createProvider_minimalConfig_v1beta1AndZone() throws IOException { } @Test - public void createProvider_missingGkeUrl_expectException() throws IOException { + public void createProvider_missingGkeUrl_expectException() + throws XdsInitializationException { CertificateProvider.DistributorWatcher distWatcher = new CertificateProvider.DistributorWatcher(); Map map = buildMissingGkeClusterUrlConfig(); @@ -189,7 +192,8 @@ public void createProvider_missingGkeUrl_expectException() throws IOException { } @Test - public void createProvider_missingGkeSaJwtLocation_expectException() throws IOException { + public void createProvider_missingGkeSaJwtLocation_expectException() + throws XdsInitializationException { CertificateProvider.DistributorWatcher distWatcher = new CertificateProvider.DistributorWatcher(); Map map = buildMissingSaJwtLocationConfig(); @@ -202,7 +206,8 @@ public void createProvider_missingGkeSaJwtLocation_expectException() throws IOEx } @Test - public void createProvider_missingProject_expectException() throws IOException { + public void createProvider_missingProject_expectException() + throws XdsInitializationException { CertificateProvider.DistributorWatcher distWatcher = new CertificateProvider.DistributorWatcher(); Map map = buildBadClusterUrlConfig(); @@ -215,7 +220,8 @@ public void createProvider_missingProject_expectException() throws IOException { } @Test - public void createProvider_badChannelCreds_expectException() throws IOException { + public void createProvider_badChannelCreds_expectException() + throws XdsInitializationException { CertificateProvider.DistributorWatcher distWatcher = new CertificateProvider.DistributorWatcher(); Map map = buildBadChannelCredsConfig(); @@ -228,7 +234,7 @@ public void createProvider_badChannelCreds_expectException() throws IOException } @Test - public void createProvider_nonDefaultFullConfig() throws IOException { + public void createProvider_nonDefaultFullConfig() throws XdsInitializationException { CertificateProvider.DistributorWatcher distWatcher = new CertificateProvider.DistributorWatcher(); Map map = buildFullConfig(); @@ -261,33 +267,37 @@ public void createProvider_nonDefaultFullConfig() throws IOException { eq(TimeUnit.SECONDS.toMillis(RPC_TIMEOUT_SECONDS))); } - private static Map buildFullConfig() throws IOException { + private static Map buildFullConfig() throws XdsInitializationException { return getCertProviderConfig(CommonCertProviderTestUtils.getNonDefaultTestBootstrapInfo()); } - private static Map buildMinimalConfig() throws IOException { + private static Map buildMinimalConfig() throws XdsInitializationException { return getCertProviderConfig(CommonCertProviderTestUtils.getMinimalBootstrapInfo()); } - private static Map buildMinimalConfig_v1beta1AndZone() throws IOException { + private static Map buildMinimalConfig_v1beta1AndZone() + throws XdsInitializationException { return getCertProviderConfig( CommonCertProviderTestUtils.getMinimalBootstrapInfo_v1beta1AndZone()); } - private static Map buildBadClusterUrlConfig() throws IOException { + private static Map buildBadClusterUrlConfig() throws XdsInitializationException { return getCertProviderConfig( CommonCertProviderTestUtils.getMinimalAndBadClusterUrlBootstrapInfo()); } - private static Map buildMissingSaJwtLocationConfig() throws IOException { + private static Map buildMissingSaJwtLocationConfig() + throws XdsInitializationException { return getCertProviderConfig(CommonCertProviderTestUtils.getMissingSaJwtLocation()); } - private static Map buildMissingGkeClusterUrlConfig() throws IOException { + private static Map buildMissingGkeClusterUrlConfig() + throws XdsInitializationException { return getCertProviderConfig(CommonCertProviderTestUtils.getMissingGkeClusterUrl()); } - private static Map buildBadChannelCredsConfig() throws IOException { + private static Map buildBadChannelCredsConfig() + throws XdsInitializationException { return getCertProviderConfig(CommonCertProviderTestUtils.getBadChannelCredsConfig()); } diff --git a/xds/src/test/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactoryTest.java b/xds/src/test/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactoryTest.java index c1c2b30e68b..7b37c2d6449 100644 --- a/xds/src/test/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactoryTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactoryTest.java @@ -31,6 +31,7 @@ import io.envoyproxy.envoy.type.matcher.v3.StringMatcher; import io.grpc.xds.Bootstrapper; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; +import io.grpc.xds.XdsInitializationException; import io.grpc.xds.internal.certprovider.CertProviderClientSslContextProvider; import io.grpc.xds.internal.certprovider.CertificateProvider; import io.grpc.xds.internal.certprovider.CertificateProviderProvider; @@ -117,7 +118,7 @@ public void createSslContextProvider_sdsConfigForCertValidationContext_expectExc } @Test - public void createCertProviderClientSslContextProvider() throws IOException { + public void createCertProviderClientSslContextProvider() throws XdsInitializationException { final CertificateProvider.DistributorWatcher[] watcherCaptor = new CertificateProvider.DistributorWatcher[1]; createAndRegisterProviderProvider(certificateProviderRegistry, watcherCaptor, "testca", 0); @@ -139,7 +140,8 @@ public void createCertProviderClientSslContextProvider() throws IOException { } @Test - public void createCertProviderClientSslContextProvider_onlyRootCert() throws IOException { + public void createCertProviderClientSslContextProvider_onlyRootCert() + throws XdsInitializationException { final CertificateProvider.DistributorWatcher[] watcherCaptor = new CertificateProvider.DistributorWatcher[1]; createAndRegisterProviderProvider(certificateProviderRegistry, watcherCaptor, "testca", 0); @@ -161,7 +163,8 @@ public void createCertProviderClientSslContextProvider_onlyRootCert() throws IOE } @Test - public void createCertProviderClientSslContextProvider_withStaticContext() throws IOException { + public void createCertProviderClientSslContextProvider_withStaticContext() + throws XdsInitializationException { final CertificateProvider.DistributorWatcher[] watcherCaptor = new CertificateProvider.DistributorWatcher[1]; createAndRegisterProviderProvider(certificateProviderRegistry, watcherCaptor, "testca", 0); @@ -190,7 +193,8 @@ public void createCertProviderClientSslContextProvider_withStaticContext() throw } @Test - public void createCertProviderClientSslContextProvider_2providers() throws IOException { + public void createCertProviderClientSslContextProvider_2providers() + throws XdsInitializationException { final CertificateProvider.DistributorWatcher[] watcherCaptor = new CertificateProvider.DistributorWatcher[2]; createAndRegisterProviderProvider(certificateProviderRegistry, watcherCaptor, "testca", 0); @@ -217,7 +221,8 @@ public void createCertProviderClientSslContextProvider_2providers() throws IOExc } @Test - public void createCertProviderClientSslContextProvider_ioException() throws IOException { + public void createCertProviderClientSslContextProvider_exception() + throws XdsInitializationException { UpstreamTlsContext upstreamTlsContext = CommonTlsContextTestsUtil.buildUpstreamTlsContextForCertProviderInstance( "gcp_id", @@ -226,12 +231,13 @@ public void createCertProviderClientSslContextProvider_ioException() throws IOEx "root-default", /* alpnProtocols= */ null, /* staticCertValidationContext= */ null); - when(bootstrapper.readBootstrap()).thenThrow(new IOException("test IOException")); + when(bootstrapper.readBootstrap()) + .thenThrow(new XdsInitializationException("test exception")); try { clientSslContextProviderFactory.create(upstreamTlsContext); Assert.fail("no exception thrown"); } catch (RuntimeException expected) { - assertThat(expected).hasMessageThat().isEqualTo("java.io.IOException: test IOException"); + assertThat(expected).hasMessageThat().contains("test exception"); } } diff --git a/xds/src/test/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactoryTest.java b/xds/src/test/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactoryTest.java index 4ab957bdb43..3f0f8cf2e04 100644 --- a/xds/src/test/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactoryTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactoryTest.java @@ -31,6 +31,7 @@ import io.envoyproxy.envoy.type.matcher.v3.StringMatcher; import io.grpc.xds.Bootstrapper; import io.grpc.xds.EnvoyServerProtoData.DownstreamTlsContext; +import io.grpc.xds.XdsInitializationException; import io.grpc.xds.internal.certprovider.CertProviderServerSslContextProvider; import io.grpc.xds.internal.certprovider.CertificateProvider; import io.grpc.xds.internal.certprovider.CertificateProviderRegistry; @@ -113,7 +114,7 @@ public void createSslContextProvider_sdsConfigForCertValidationContext_expectExc } @Test - public void createCertProviderServerSslContextProvider() throws IOException { + public void createCertProviderServerSslContextProvider() throws XdsInitializationException { final CertificateProvider.DistributorWatcher[] watcherCaptor = new CertificateProvider.DistributorWatcher[1]; createAndRegisterProviderProvider(certificateProviderRegistry, watcherCaptor, "testca", 0); @@ -136,7 +137,8 @@ public void createCertProviderServerSslContextProvider() throws IOException { } @Test - public void createCertProviderServerSslContextProvider_onlyCertInstance() throws IOException { + public void createCertProviderServerSslContextProvider_onlyCertInstance() + throws XdsInitializationException { final CertificateProvider.DistributorWatcher[] watcherCaptor = new CertificateProvider.DistributorWatcher[1]; createAndRegisterProviderProvider(certificateProviderRegistry, watcherCaptor, "testca", 0); @@ -159,7 +161,8 @@ public void createCertProviderServerSslContextProvider_onlyCertInstance() throws } @Test - public void createCertProviderServerSslContextProvider_withStaticContext() throws IOException { + public void createCertProviderServerSslContextProvider_withStaticContext() + throws XdsInitializationException { final CertificateProvider.DistributorWatcher[] watcherCaptor = new CertificateProvider.DistributorWatcher[1]; createAndRegisterProviderProvider(certificateProviderRegistry, watcherCaptor, "testca", 0); @@ -189,7 +192,8 @@ public void createCertProviderServerSslContextProvider_withStaticContext() throw } @Test - public void createCertProviderServerSslContextProvider_2providers() throws IOException { + public void createCertProviderServerSslContextProvider_2providers() + throws XdsInitializationException { final CertificateProvider.DistributorWatcher[] watcherCaptor = new CertificateProvider.DistributorWatcher[2]; createAndRegisterProviderProvider(certificateProviderRegistry, watcherCaptor, "testca", 0); @@ -217,7 +221,8 @@ public void createCertProviderServerSslContextProvider_2providers() throws IOExc } @Test - public void createCertProviderServerSslContextProvider_ioException() throws IOException { + public void createCertProviderServerSslContextProvider_exception() + throws XdsInitializationException { DownstreamTlsContext downstreamTlsContext = CommonTlsContextTestsUtil.buildDownstreamTlsContextForCertProviderInstance( "gcp_id", @@ -227,12 +232,13 @@ public void createCertProviderServerSslContextProvider_ioException() throws IOEx /* alpnProtocols= */ null, /* staticCertValidationContext= */ null, /* requireClientCert= */ true); - when(bootstrapper.readBootstrap()).thenThrow(new IOException("test IOException")); + when(bootstrapper.readBootstrap()) + .thenThrow(new XdsInitializationException("test exception")); try { serverSslContextProviderFactory.create(downstreamTlsContext); Assert.fail("no exception thrown"); } catch (RuntimeException expected) { - assertThat(expected).hasMessageThat().isEqualTo("java.io.IOException: test IOException"); + assertThat(expected).hasMessageThat().contains("test exception"); } } From ee5b5929d56de13e00c88352cd2ad16ffd6e0308 Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Tue, 15 Sep 2020 11:22:44 -0700 Subject: [PATCH 42/86] api,netty: Fix TruthIncompatibleType --- api/src/test/java/io/grpc/ClientInterceptorsTest.java | 2 +- .../io/grpc/netty/WriteBufferingAndExceptionHandlerTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/test/java/io/grpc/ClientInterceptorsTest.java b/api/src/test/java/io/grpc/ClientInterceptorsTest.java index df5e8007582..555e9405764 100644 --- a/api/src/test/java/io/grpc/ClientInterceptorsTest.java +++ b/api/src/test/java/io/grpc/ClientInterceptorsTest.java @@ -329,7 +329,7 @@ public ClientCall interceptCall( assertSame(listener, call.listener); assertSame(headers, call.headers); interceptedCall.sendMessage(null /*request*/); - assertThat(call.messages).containsExactly((Void) null /*request*/); + assertThat(call.messages).containsExactly((String) null); interceptedCall.halfClose(); assertTrue(call.halfClosed); interceptedCall.request(1); diff --git a/netty/src/test/java/io/grpc/netty/WriteBufferingAndExceptionHandlerTest.java b/netty/src/test/java/io/grpc/netty/WriteBufferingAndExceptionHandlerTest.java index b3b536b20ea..b99a9386fcf 100644 --- a/netty/src/test/java/io/grpc/netty/WriteBufferingAndExceptionHandlerTest.java +++ b/netty/src/test/java/io/grpc/netty/WriteBufferingAndExceptionHandlerTest.java @@ -345,7 +345,7 @@ public void run() { assertThat(chan.pipeline().context(handler)).isNull(); assertThat(write.get().getClass()).isSameInstanceAs(Object.class); assertTrue(flush.get()); - assertThat(chan.pipeline()).doesNotContain(handler); + assertThat(chan.pipeline().toMap().values()).doesNotContain(handler); } @Test From 5879b53c573549028dbcf84fdc206503311870a6 Mon Sep 17 00:00:00 2001 From: Russell Shaw <69813534+kiwi1969@users.noreply.github.com> Date: Wed, 16 Sep 2020 11:07:52 -0400 Subject: [PATCH 43/86] netty: Add support for IBMJSSE2 (#7422) This is a very simple change to test for IBMJSSE2 security provider in addition to the others. IBM JRE does not support the Sun provider, but instead has IBMJSSE2 which supports the same API calls. I tested this on Z/OS machine as now it works when before it couldn't find a security provider --- .../main/java/io/grpc/netty/GrpcSslContexts.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/netty/src/main/java/io/grpc/netty/GrpcSslContexts.java b/netty/src/main/java/io/grpc/netty/GrpcSslContexts.java index 9be8f9e849c..38baa81f7a7 100644 --- a/netty/src/main/java/io/grpc/netty/GrpcSslContexts.java +++ b/netty/src/main/java/io/grpc/netty/GrpcSslContexts.java @@ -82,6 +82,7 @@ private GrpcSslContexts() {} NEXT_PROTOCOL_VERSIONS); private static final String SUN_PROVIDER_NAME = "SunJSSE"; + private static final String IBM_PROVIDER_NAME = "IBMJSSE2"; /** * Creates an SslContextBuilder with ciphers and APN appropriate for gRPC. @@ -196,7 +197,14 @@ public static SslContextBuilder configure(SslContextBuilder builder, Provider jd apc = ALPN; } else { throw new IllegalArgumentException( - SUN_PROVIDER_NAME + " selected, but Java 9+ and Jetty NPN/ALPN unavailable"); + jdkProvider.getName() + " selected, but Java 9+ and Jetty NPN/ALPN unavailable"); + } + } else if (IBM_PROVIDER_NAME.equals(jdkProvider.getName())) { + if (JettyTlsUtil.isJava9AlpnAvailable()) { + apc = ALPN; + } else { + throw new IllegalArgumentException( + jdkProvider.getName() + " selected, but Java 9+ ALPN unavailable"); } } else if (ConscryptLoader.isConscrypt(jdkProvider)) { apc = ALPN; @@ -243,6 +251,10 @@ private static Provider findJdkProvider() { || JettyTlsUtil.isJava9AlpnAvailable()) { return provider; } + } else if (IBM_PROVIDER_NAME.equals(provider.getName())) { + if (JettyTlsUtil.isJava9AlpnAvailable()) { + return provider; + } } else if (ConscryptLoader.isConscrypt(provider)) { return provider; } From cccd940e43f5871af94b1b1d05cbd97b000d99e7 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Wed, 16 Sep 2020 11:38:55 -0400 Subject: [PATCH 44/86] okhttp: cleanup channel in channel builder tests --- .../java/io/grpc/okhttp/OkHttpChannelBuilderTest.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java b/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java index 60ec5da1c4e..1316463f7a3 100644 --- a/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java +++ b/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java @@ -24,10 +24,12 @@ import com.squareup.okhttp.ConnectionSpec; import io.grpc.ChannelLogger; +import io.grpc.ManagedChannel; import io.grpc.internal.ClientTransportFactory; import io.grpc.internal.FakeClock; import io.grpc.internal.GrpcUtil; import io.grpc.internal.SharedResourceHolder; +import io.grpc.testing.GrpcCleanupRule; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.Socket; @@ -46,11 +48,13 @@ public class OkHttpChannelBuilderTest { @Rule public final ExpectedException thrown = ExpectedException.none(); + @Rule public final GrpcCleanupRule grpcCleanupRule = new GrpcCleanupRule(); @Test public void authorityIsReadable() { OkHttpChannelBuilder builder = OkHttpChannelBuilder.forAddress("original", 1234); - assertEquals("original:1234", builder.build().authority()); + ManagedChannel channel = grpcCleanupRule.register(builder.build()); + assertEquals("original:1234", channel.authority()); } @Test @@ -68,7 +72,8 @@ public void overrideAuthorityIsReadableForTarget() { private void overrideAuthorityIsReadableHelper(OkHttpChannelBuilder builder, String overrideAuthority) { builder.overrideAuthority(overrideAuthority); - assertEquals(overrideAuthority, builder.build().authority()); + ManagedChannel channel = grpcCleanupRule.register(builder.build()); + assertEquals(overrideAuthority, channel.authority()); } @Test From eb871698e3df8d4ec42e1266760927b44adc4b65 Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Wed, 16 Sep 2020 10:07:52 -0700 Subject: [PATCH 45/86] Revert "SECURITY.md: add instruction for disabling Conscrypt's default TrustManager (#6962)" (#7428) This reverts commit e089ceaadca78029c7c984eeb0ff199b2fbd78b4. --- SECURITY.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 5aeff135349..631b4396fce 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -232,14 +232,9 @@ import java.security.Security; ... // Somewhere in main() -Security.insertProviderAt( - Conscrypt.newProviderBuilder().provideTrustManager(false).build(), 1); +Security.insertProviderAt(Conscrypt.newProvider(), 1); ``` -Note: according to [Conscrypt Implementation Notes](https://github.com/google/conscrypt/blob/2.4.0/IMPLEMENTATION_NOTES.md#hostname-verification), -its default `HostnameVerifier` on OpenJDK always fails. This can be worked -around by disabling its default `TrustManager` implementation as shown above. - ### TLS with Jetty ALPN **Please do not use Jetty ALPN** From f04f33efe8e4f4d88099df68b6072416da468ad6 Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Wed, 16 Sep 2020 10:51:38 -0700 Subject: [PATCH 46/86] xds: resource (with version info) should persist across ADS streams along with XdsClient lifetime (#7427) The version_info in the xDS protocol represents the client's knowledge for the state of that resource type. It should persist across ADS stream recreation. Even if the ADS stream is recreated, the XdsClient should persist its knowledge for resources it has received. With this implementation, client and server are stateful across the xDS communication. With persisted version_info, the management server knows resources that the client currently knows even after the stream is recreated. So it does not need to re-send resources that the client received with the previous stream. --- .../main/java/io/grpc/xds/XdsClientImpl.java | 30 ++++++---------- .../java/io/grpc/xds/XdsClientImplTest.java | 35 +++++++++--------- .../xds/XdsClientImplTestForListener.java | 12 ++++--- .../java/io/grpc/xds/XdsClientImplTestV2.java | 36 ++++++++++--------- 4 files changed, 55 insertions(+), 58 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/XdsClientImpl.java b/xds/src/main/java/io/grpc/xds/XdsClientImpl.java index 82addf72b3e..1dcb0c656cc 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientImpl.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientImpl.java @@ -163,6 +163,14 @@ final class XdsClientImpl extends XdsClient { private final LoadStatsManager loadStatsManager = new LoadStatsManager(); + // Last successfully applied version_info for each resource type. Starts with empty string. + // A version_info is used to update management server with client's most recent knowledge of + // resources. + private String ldsVersion = ""; + private String rdsVersion = ""; + private String cdsVersion = ""; + private String edsVersion = ""; + // Timer for concluding the currently requesting LDS resource not found. @Nullable private ScheduledHandle ldsRespTimer; @@ -226,7 +234,7 @@ void shutdown() { if (adsStream != null) { adsStream.close(Status.CANCELLED.withDescription("shutdown").asException()); } - cleanUpResources(); + cleanUpResourceTimers(); if (lrsClient != null) { lrsClient.stopLoadReporting(); lrsClient = null; @@ -236,15 +244,7 @@ void shutdown() { } } - /** - * Purge cache for resources and cancel resource fetch timers. - */ - private void cleanUpResources() { - clusterNamesToClusterUpdates.clear(); - absentCdsResources.clear(); - clusterNamesToEndpointUpdates.clear(); - absentEdsResources.clear(); - + private void cleanUpResourceTimers() { if (ldsRespTimer != null) { ldsRespTimer.cancel(); ldsRespTimer = null; @@ -1434,14 +1434,6 @@ private abstract class AbstractAdsStream { private boolean responseReceived; private boolean closed; - // Last successfully applied version_info for each resource type. Starts with empty string. - // A version_info is used to update management server with client's most recent knowledge of - // resources. - private String ldsVersion = ""; - private String rdsVersion = ""; - private String cdsVersion = ""; - private String edsVersion = ""; - // Response nonce for the most recently received discovery responses of each resource type. // Client initiated requests start response nonce with empty string. // A nonce is used to indicate the specific DiscoveryResponse each DiscoveryRequest @@ -1558,7 +1550,7 @@ private void handleStreamClosed(Status error) { } } cleanUp(); - cleanUpResources(); + cleanUpResourceTimers(); if (responseReceived || retryBackoffPolicy == null) { // Reset the backoff sequence if had received a response, or backoff sequence // has never been initialized. diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java index b55a649741c..a9b8e42b6bb 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java @@ -2672,7 +2672,7 @@ public void streamClosedAndRetryWhenResolvingConfig() { responseObserver = responseObserverCaptor.getValue(); requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequest(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS, ""))); // RPC stream closed immediately @@ -2689,10 +2689,11 @@ public void streamClosedAndRetryWhenResolvingConfig() { responseObserver = responseObserverCaptor.getValue(); requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequest(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS, ""))); // Management server sends an LDS response. + ldsResponse = buildDiscoveryResponse("1", listeners, XdsClientImpl.ADS_TYPE_URL_LDS, "0001"); responseObserver.onNext(ldsResponse); // Client sends an ACK LDS request and an RDS request for "route-foo.googleapis.com". (Omitted) @@ -2724,7 +2725,7 @@ public void streamClosedAndRetryWhenResolvingConfig() { fakeClock.runDueTasks(); requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequest(NODE, "1", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS, ""))); verifyNoMoreInteractions(backoffPolicyProvider, backoffPolicy1, backoffPolicy2); @@ -2786,7 +2787,7 @@ public void streamClosedAndRetry() { // Retry resumes requests for all wanted resources. verify(requestObserver) - .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequest(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", @@ -2815,7 +2816,7 @@ public void streamClosedAndRetry() { responseObserver = responseObserverCaptor.getValue(); requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequest(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", @@ -2844,7 +2845,7 @@ public void streamClosedAndRetry() { responseObserver = responseObserverCaptor.getValue(); requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequest(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", @@ -2877,10 +2878,10 @@ public void streamClosedAndRetry() { requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequest(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS, ""))); verify(requestObserver) - .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", + .onNext(eq(buildDiscoveryRequest(NODE, "0", "cluster.googleapis.com", XdsClientImpl.ADS_TYPE_URL_CDS, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", @@ -2905,10 +2906,10 @@ public void streamClosedAndRetry() { .streamAggregatedResources(responseObserverCaptor.capture()); requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequest(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS, ""))); verify(requestObserver) - .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", + .onNext(eq(buildDiscoveryRequest(NODE, "0", "cluster.googleapis.com", XdsClientImpl.ADS_TYPE_URL_CDS, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", @@ -2950,7 +2951,7 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { StreamObserver requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequest(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS, ""))); // Management server becomes unreachable. @@ -2971,7 +2972,7 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequest(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", @@ -2995,7 +2996,7 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequest(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", @@ -3038,10 +3039,10 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { responseObserver = responseObserverCaptor.getValue(); requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequest(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS, ""))); verify(requestObserver) - .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", + .onNext(eq(buildDiscoveryRequest(NODE, "0", "cluster.googleapis.com", XdsClientImpl.ADS_TYPE_URL_CDS, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster2.googleapis.com", @@ -3065,10 +3066,10 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequest(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequest(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS, ""))); verify(requestObserver, never()) - .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster.googleapis.com", + .onNext(eq(buildDiscoveryRequest(NODE, "0", "cluster.googleapis.com", XdsClientImpl.ADS_TYPE_URL_CDS, ""))); verify(requestObserver, never()) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster2.googleapis.com", diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTestForListener.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTestForListener.java index ba729f04816..77c073cf7b6 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientImplTestForListener.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTestForListener.java @@ -708,6 +708,8 @@ public void streamClosedAndRetry() { buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); responseObserver.onNext(response); + // Client sent an ACK CDS request (Omitted). + ArgumentCaptor statusCaptor = ArgumentCaptor.forClass(null); // Management server closes the RPC stream with an error. @@ -725,7 +727,7 @@ public void streamClosedAndRetry() { // Retry resumes requests for all wanted resources. verify(requestObserver) - .onNext(eq(buildDiscoveryRequest(getNodeToVerify(), "", + .onNext(eq(buildDiscoveryRequest(getNodeToVerify(), "0", XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); // Management server becomes unreachable. @@ -744,7 +746,7 @@ public void streamClosedAndRetry() { responseObserver = responseObserverCaptor.getValue(); requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequest(getNodeToVerify(), "", + .onNext(eq(buildDiscoveryRequest(getNodeToVerify(), "0", XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); // Management server is still not reachable. @@ -763,7 +765,7 @@ public void streamClosedAndRetry() { responseObserver = responseObserverCaptor.getValue(); requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequest(getNodeToVerify(), "", + .onNext(eq(buildDiscoveryRequest(getNodeToVerify(), "0", XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); // Management server sends back a LDS response. @@ -786,7 +788,7 @@ public void streamClosedAndRetry() { requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequest(getNodeToVerify(), "", + .onNext(eq(buildDiscoveryRequest(getNodeToVerify(), "1", XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); // Management server becomes unreachable again. @@ -804,7 +806,7 @@ public void streamClosedAndRetry() { .streamAggregatedResources(responseObserverCaptor.capture()); requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequest(getNodeToVerify(), "", + .onNext(eq(buildDiscoveryRequest(getNodeToVerify(), "1", XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); verifyNoMoreInteractions(mockedDiscoveryService, backoffPolicyProvider, backoffPolicy1, diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java index d3691a9a475..51b5b5db684 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java @@ -2682,7 +2682,7 @@ public void streamClosedAndRetryWhenResolvingConfig() { responseObserver = responseObserverCaptor.getValue(); requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); // RPC stream closed immediately @@ -2699,10 +2699,12 @@ public void streamClosedAndRetryWhenResolvingConfig() { responseObserver = responseObserverCaptor.getValue(); requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); // Management server sends an LDS response. + ldsResponse = + buildDiscoveryResponseV2("1", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0001"); responseObserver.onNext(ldsResponse); // Client sends an ACK LDS request and an RDS request for "route-foo.googleapis.com". (Omitted) @@ -2734,7 +2736,7 @@ public void streamClosedAndRetryWhenResolvingConfig() { fakeClock.runDueTasks(); requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequestV2(NODE, "1", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); verifyNoMoreInteractions(backoffPolicyProvider, backoffPolicy1, backoffPolicy2); @@ -2796,7 +2798,7 @@ public void streamClosedAndRetry() { // Retry resumes requests for all wanted resources. verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", @@ -2825,7 +2827,7 @@ public void streamClosedAndRetry() { responseObserver = responseObserverCaptor.getValue(); requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", @@ -2854,7 +2856,7 @@ public void streamClosedAndRetry() { responseObserver = responseObserverCaptor.getValue(); requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", @@ -2887,10 +2889,10 @@ public void streamClosedAndRetry() { requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster.googleapis.com", XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", @@ -2915,10 +2917,10 @@ public void streamClosedAndRetry() { .streamAggregatedResources(responseObserverCaptor.capture()); requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster.googleapis.com", XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", @@ -2960,7 +2962,7 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { StreamObserver requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); // Management server becomes unreachable. @@ -2981,7 +2983,7 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", @@ -3005,7 +3007,7 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", @@ -3048,10 +3050,10 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { responseObserver = responseObserverCaptor.getValue(); requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster.googleapis.com", XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); verify(requestObserver) .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster2.googleapis.com", @@ -3075,10 +3077,10 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { requestObserver = requestObservers.poll(); verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); verify(requestObserver, never()) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", + .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster.googleapis.com", XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); verify(requestObserver, never()) .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster2.googleapis.com", From 80f6d874cf415cc470eba96efeea52a87bb5d1f9 Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Wed, 16 Sep 2020 15:32:20 -0700 Subject: [PATCH 47/86] core: lint remove redundant field (#7433) --- core/src/main/java/io/grpc/internal/ManagedChannelImpl.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java b/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java index 494899b48c0..1bd42c04e8d 100644 --- a/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java +++ b/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java @@ -1737,7 +1737,6 @@ private final class SubchannelImpl extends AbstractSubchannel { final InternalLogId subchannelLogId; final ChannelLoggerImpl subchannelLogger; final ChannelTracer subchannelTracer; - SubchannelStateListener listener; InternalSubchannel subchannel; boolean started; boolean shutdown; @@ -1759,7 +1758,6 @@ private void internalStart(final SubchannelStateListener listener) { checkState(!started, "already started"); checkState(!shutdown, "already shutdown"); started = true; - this.listener = listener; // TODO(zhangkun): possibly remove the volatile of terminating when this whole method is // required to be called from syncContext if (terminating) { From 3abdb2859f4f9d3576fe98d50a6f59eaff20c3b5 Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Wed, 16 Sep 2020 16:48:22 -0700 Subject: [PATCH 48/86] grpclb: cache requestConnection if no subchannel created An issue was found during CBT RLS client testing: The RLS lb creates grplb child balancer, calls `grpclb.handleResolvedAddress()` then immediately calls `grpclb.requestConnection()`. The subchannel in `GrpclbState.currentPicker.pickList` contains only `GrpclbState.BUFFER_ENTRY` at the moment `grpclb.requestConnection()` is called, and therefore the `requestConnection()` is no-op, and RPC is hanging. --- grpclb/src/main/java/io/grpc/grpclb/GrpclbState.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/grpclb/src/main/java/io/grpc/grpclb/GrpclbState.java b/grpclb/src/main/java/io/grpc/grpclb/GrpclbState.java index b6c99135e22..3630db63a16 100644 --- a/grpclb/src/main/java/io/grpc/grpclb/GrpclbState.java +++ b/grpclb/src/main/java/io/grpc/grpclb/GrpclbState.java @@ -158,6 +158,7 @@ enum Mode { private List backendList = Collections.emptyList(); private RoundRobinPicker currentPicker = new RoundRobinPicker(Collections.emptyList(), Arrays.asList(BUFFER_ENTRY)); + private boolean requestConnectionPending; GrpclbState( GrpclbConfig config, @@ -242,9 +243,11 @@ void handleAddresses( } void requestConnection() { + requestConnectionPending = true; for (RoundRobinEntry entry : currentPicker.pickList) { if (entry instanceof IdleSubchannelEntry) { ((IdleSubchannelEntry) entry).subchannel.requestConnection(); + requestConnectionPending = false; } } } @@ -471,6 +474,10 @@ public void onSubchannelState(ConnectivityStateInfo newState) { handleSubchannelState(subchannel, newState); } }); + if (requestConnectionPending) { + subchannel.requestConnection(); + requestConnectionPending = false; + } } else { subchannel = subchannels.values().iterator().next(); subchannel.updateAddresses(eagList); From 92cbc578a2f3cbe0c5c6129ee9cd50a75a8a90dc Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Thu, 17 Sep 2020 21:08:54 -0700 Subject: [PATCH 49/86] xds: remove logging in data path (#7437) --- xds/src/main/java/io/grpc/xds/XdsNameResolver.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java index 22f74f7f795..f6e00cd85fb 100644 --- a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java +++ b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java @@ -238,10 +238,6 @@ public Result selectConfig(PickSubchannelArgs args) { Map rawServiceConfig = generateServiceConfigWithMethodTimeoutConfig( selectedRoute.getRouteAction().getTimeoutNano()); - if (logger.isLoggable(XdsLogLevel.INFO)) { - logger.log(XdsLogLevel.INFO, - "Generated service config (method config):\n{0}", new Gson().toJson(rawServiceConfig)); - } ConfigOrError parsedServiceConfig = serviceConfigParser.parseServiceConfig(rawServiceConfig); Object config = parsedServiceConfig.getConfig(); if (config == null) { From 04871dcc2a106b2504489af1c1f6d53a38a0ff41 Mon Sep 17 00:00:00 2001 From: sanjaypujare Date: Thu, 17 Sep 2020 21:25:19 -0700 Subject: [PATCH 50/86] xds: bootstrapper fixes: remove extra readBootstrap & avoid parseConfig (#7436) --- .../main/java/io/grpc/xds/Bootstrapper.java | 2 +- .../sds/ClientSslContextProviderFactory.java | 11 +- .../sds/ServerSslContextProviderFactory.java | 11 +- .../internal/sds/TlsContextManagerImpl.java | 10 +- .../grpc/xds/CommonBootstrapperTestUtils.java | 74 +++++ .../io/grpc/xds/XdsSdsClientServerTest.java | 4 +- ...tProviderClientSslContextProviderTest.java | 11 +- ...tProviderServerSslContextProviderTest.java | 11 +- .../CommonCertProviderTestUtils.java | 270 ------------------ ...MeshCaCertificateProviderProviderTest.java | 202 +++++++++---- .../ClientSslContextProviderFactoryTest.java | 10 +- .../ServerSslContextProviderFactoryTest.java | 10 +- 12 files changed, 269 insertions(+), 357 deletions(-) create mode 100644 xds/src/test/java/io/grpc/xds/CommonBootstrapperTestUtils.java diff --git a/xds/src/main/java/io/grpc/xds/Bootstrapper.java b/xds/src/main/java/io/grpc/xds/Bootstrapper.java index 19856b88a66..f1f10a5d707 100644 --- a/xds/src/main/java/io/grpc/xds/Bootstrapper.java +++ b/xds/src/main/java/io/grpc/xds/Bootstrapper.java @@ -84,7 +84,7 @@ public static Bootstrapper getInstance() { /** Parses a raw string into {@link BootstrapInfo}. */ @VisibleForTesting @SuppressWarnings("unchecked") - public static BootstrapInfo parseConfig(String rawData) throws XdsInitializationException { + static BootstrapInfo parseConfig(String rawData) throws XdsInitializationException { XdsLogger logger = XdsLogger.withPrefix(LOG_PREFIX); logger.log(XdsLogLevel.INFO, "Reading bootstrap information"); Map rawBootstrap; diff --git a/xds/src/main/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactory.java b/xds/src/main/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactory.java index 1163e94ab28..19ad67186c3 100644 --- a/xds/src/main/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactory.java +++ b/xds/src/main/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactory.java @@ -34,8 +34,8 @@ final class ClientSslContextProviderFactory private final CertProviderClientSslContextProvider.Factory certProviderClientSslContextProviderFactory; - ClientSslContextProviderFactory() { - this(Bootstrapper.getInstance(), CertProviderClientSslContextProvider.Factory.getInstance()); + ClientSslContextProviderFactory(Bootstrapper bootstrapper) { + this(bootstrapper, CertProviderClientSslContextProvider.Factory.getInstance()); } ClientSslContextProviderFactory( @@ -58,7 +58,7 @@ public SslContextProvider create(UpstreamTlsContext upstreamTlsContext) { try { return SdsClientSslContextProvider.getProvider( upstreamTlsContext, - Bootstrapper.getInstance().readBootstrap().getNode().toEnvoyProtoNodeV2(), + bootstrapper.readBootstrap().getNode().toEnvoyProtoNodeV2(), Executors.newSingleThreadExecutor(new ThreadFactoryBuilder() .setNameFormat("client-sds-sslcontext-provider-%d") .setDaemon(true) @@ -70,10 +70,11 @@ public SslContextProvider create(UpstreamTlsContext upstreamTlsContext) { } else if (CommonTlsContextUtil.hasCertProviderInstance( upstreamTlsContext.getCommonTlsContext())) { try { + Bootstrapper.BootstrapInfo bootstrapInfo = bootstrapper.readBootstrap(); return certProviderClientSslContextProviderFactory.getProvider( upstreamTlsContext, - bootstrapper.readBootstrap().getNode().toEnvoyProtoNode(), - bootstrapper.readBootstrap().getCertProviders()); + bootstrapInfo.getNode().toEnvoyProtoNode(), + bootstrapInfo.getCertProviders()); } catch (XdsInitializationException e) { throw new RuntimeException(e); } diff --git a/xds/src/main/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactory.java b/xds/src/main/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactory.java index e3f006acee6..3a33c975c0b 100644 --- a/xds/src/main/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactory.java +++ b/xds/src/main/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactory.java @@ -34,8 +34,8 @@ final class ServerSslContextProviderFactory private final CertProviderServerSslContextProvider.Factory certProviderServerSslContextProviderFactory; - ServerSslContextProviderFactory() { - this(Bootstrapper.getInstance(), CertProviderServerSslContextProvider.Factory.getInstance()); + ServerSslContextProviderFactory(Bootstrapper bootstrapper) { + this(bootstrapper, CertProviderServerSslContextProvider.Factory.getInstance()); } ServerSslContextProviderFactory( @@ -60,7 +60,7 @@ public SslContextProvider create( try { return SdsServerSslContextProvider.getProvider( downstreamTlsContext, - Bootstrapper.getInstance().readBootstrap().getNode().toEnvoyProtoNodeV2(), + bootstrapper.readBootstrap().getNode().toEnvoyProtoNodeV2(), Executors.newSingleThreadExecutor(new ThreadFactoryBuilder() .setNameFormat("server-sds-sslcontext-provider-%d") .setDaemon(true) @@ -72,10 +72,11 @@ public SslContextProvider create( } else if (CommonTlsContextUtil.hasCertProviderInstance( downstreamTlsContext.getCommonTlsContext())) { try { + Bootstrapper.BootstrapInfo bootstrapInfo = bootstrapper.readBootstrap(); return certProviderServerSslContextProviderFactory.getProvider( downstreamTlsContext, - bootstrapper.readBootstrap().getNode().toEnvoyProtoNode(), - bootstrapper.readBootstrap().getCertProviders()); + bootstrapInfo.getNode().toEnvoyProtoNode(), + bootstrapInfo.getCertProviders()); } catch (XdsInitializationException e) { throw new RuntimeException(e); } diff --git a/xds/src/main/java/io/grpc/xds/internal/sds/TlsContextManagerImpl.java b/xds/src/main/java/io/grpc/xds/internal/sds/TlsContextManagerImpl.java index a870e9d36fd..31d2e014ee9 100644 --- a/xds/src/main/java/io/grpc/xds/internal/sds/TlsContextManagerImpl.java +++ b/xds/src/main/java/io/grpc/xds/internal/sds/TlsContextManagerImpl.java @@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import com.google.common.annotations.VisibleForTesting; +import io.grpc.xds.Bootstrapper; import io.grpc.xds.EnvoyServerProtoData.DownstreamTlsContext; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import io.grpc.xds.internal.sds.ReferenceCountingMap.ValueFactory; @@ -36,8 +37,11 @@ public final class TlsContextManagerImpl implements TlsContextManager { private final ReferenceCountingMap mapForClients; private final ReferenceCountingMap mapForServers; - private TlsContextManagerImpl() { - this(new ClientSslContextProviderFactory(), new ServerSslContextProviderFactory()); + /** Create a TlsContextManagerImpl instance using the passed in {@link Bootstrapper}. */ + @VisibleForTesting public TlsContextManagerImpl(Bootstrapper bootstrapper) { + this( + new ClientSslContextProviderFactory(bootstrapper), + new ServerSslContextProviderFactory(bootstrapper)); } @VisibleForTesting @@ -53,7 +57,7 @@ private TlsContextManagerImpl() { /** Gets the TlsContextManagerImpl singleton. */ public static synchronized TlsContextManagerImpl getInstance() { if (instance == null) { - instance = new TlsContextManagerImpl(); + instance = new TlsContextManagerImpl(Bootstrapper.getInstance()); } return instance; } diff --git a/xds/src/test/java/io/grpc/xds/CommonBootstrapperTestUtils.java b/xds/src/test/java/io/grpc/xds/CommonBootstrapperTestUtils.java new file mode 100644 index 00000000000..547c61f94ca --- /dev/null +++ b/xds/src/test/java/io/grpc/xds/CommonBootstrapperTestUtils.java @@ -0,0 +1,74 @@ +/* + * Copyright 2020 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.xds; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.grpc.internal.JsonParser; +import java.io.IOException; +import java.util.Map; + +public class CommonBootstrapperTestUtils { + private static final String FILE_WATCHER_CONFIG = "{\"path\": \"/etc/secret/certs\"}"; + private static final String MESHCA_CONFIG = + "{\n" + + " \"server\": {\n" + + " \"api_type\": \"GRPC\",\n" + + " \"grpc_services\": [{\n" + + " \"google_grpc\": {\n" + + " \"target_uri\": \"meshca.com\",\n" + + " \"channel_credentials\": {\"google_default\": {}},\n" + + " \"call_credentials\": [{\n" + + " \"sts_service\": {\n" + + " \"token_exchange_service\": \"securetoken.googleapis.com\",\n" + + " \"subject_token_path\": \"/etc/secret/sajwt.token\"\n" + + " }\n" + + " }]\n" // end call_credentials + + " },\n" // end google_grpc + + " \"time_out\": {\"seconds\": 10}\n" + + " }]\n" // end grpc_services + + " },\n" // end server + + " \"certificate_lifetime\": {\"seconds\": 86400},\n" + + " \"renewal_grace_period\": {\"seconds\": 3600},\n" + + " \"key_type\": \"RSA\",\n" + + " \"key_size\": 2048,\n" + + " \"location\": \"https://container.googleapis.com/v1/project/test-project1/locations/test-zone2/clusters/test-cluster3\"\n" + + " }"; + + /** Creates a test bootstrap info object. */ + @SuppressWarnings("unchecked") + public static Bootstrapper.BootstrapInfo getTestBootstrapInfo() { + try { + Bootstrapper.CertificateProviderInfo gcpId = + new Bootstrapper.CertificateProviderInfo( + "testca", (Map) JsonParser.parse(MESHCA_CONFIG)); + Bootstrapper.CertificateProviderInfo fileProvider = + new Bootstrapper.CertificateProviderInfo( + "file_watcher", (Map) JsonParser.parse(FILE_WATCHER_CONFIG)); + Map certProviders = + ImmutableMap.of("gcp_id", gcpId, "file_provider", fileProvider); + Bootstrapper.BootstrapInfo bootstrapInfo = + new Bootstrapper.BootstrapInfo( + ImmutableList.of(), + EnvoyProtoData.Node.newBuilder().build(), + certProviders); + return bootstrapInfo; + } catch (IOException e) { + throw new AssertionError(e); + } + } +} diff --git a/xds/src/test/java/io/grpc/xds/XdsSdsClientServerTest.java b/xds/src/test/java/io/grpc/xds/XdsSdsClientServerTest.java index 3ccf551756e..0a16e3f60e5 100644 --- a/xds/src/test/java/io/grpc/xds/XdsSdsClientServerTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsSdsClientServerTest.java @@ -80,10 +80,12 @@ public class XdsSdsClientServerTest { @Rule public final GrpcCleanupRule cleanupRule = new GrpcCleanupRule(); private int port; private FakeNameResolverFactory fakeNameResolverFactory; + private Bootstrapper mockBootstrapper; @Before public void setUp() throws IOException { port = XdsServerTestHelper.findFreePort(); + mockBootstrapper = mock(Bootstrapper.class); } @After @@ -367,7 +369,7 @@ private SimpleServiceGrpc.SimpleServiceBlockingStub getBlockingStub( ? Attributes.newBuilder() .set(XdsAttributes.ATTR_SSL_CONTEXT_PROVIDER_SUPPLIER, new SslContextProviderSupplier( - upstreamTlsContext, TlsContextManagerImpl.getInstance())) + upstreamTlsContext, new TlsContextManagerImpl(mockBootstrapper))) .build() : Attributes.EMPTY; fakeNameResolverFactory.setServers( diff --git a/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderClientSslContextProviderTest.java b/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderClientSslContextProviderTest.java index f7d8d69bfa1..00b29014648 100644 --- a/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderClientSslContextProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderClientSslContextProviderTest.java @@ -34,6 +34,7 @@ import io.envoyproxy.envoy.config.core.v3.DataSource; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CertificateValidationContext; import io.grpc.xds.Bootstrapper; +import io.grpc.xds.CommonBootstrapperTestUtils; import io.grpc.xds.EnvoyServerProtoData; import io.grpc.xds.internal.sds.CommonTlsContextTestsUtil; import io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.TestCallback; @@ -96,7 +97,7 @@ public void testProviderForClient_mtls() throws Exception { getSslContextProvider( "gcp_id", "gcp_id", - CommonCertProviderTestUtils.getTestBootstrapInfo(), + CommonBootstrapperTestUtils.getTestBootstrapInfo(), /* alpnProtocols= */ null, /* staticCertValidationContext= */ null); @@ -159,7 +160,7 @@ public void testProviderForClient_queueExecutor() throws Exception { getSslContextProvider( "gcp_id", "gcp_id", - CommonCertProviderTestUtils.getTestBootstrapInfo(), + CommonBootstrapperTestUtils.getTestBootstrapInfo(), /* alpnProtocols= */ null, /* staticCertValidationContext= */ null); QueuedExecutor queuedExecutor = new QueuedExecutor(); @@ -192,7 +193,7 @@ public void testProviderForClient_tls() throws Exception { getSslContextProvider( /* certInstanceName= */ null, "gcp_id", - CommonCertProviderTestUtils.getTestBootstrapInfo(), + CommonBootstrapperTestUtils.getTestBootstrapInfo(), /* alpnProtocols= */ null, /* staticCertValidationContext= */ null); @@ -229,7 +230,7 @@ public void testProviderForClient_sslContextException_onError() throws Exception getSslContextProvider( /* certInstanceName= */ null, "gcp_id", - CommonCertProviderTestUtils.getTestBootstrapInfo(), + CommonBootstrapperTestUtils.getTestBootstrapInfo(), /* alpnProtocols= */null, staticCertValidationContext); @@ -260,7 +261,7 @@ public void testProviderForClient_rootInstanceNull_expectError() throws Exceptio getSslContextProvider( /* certInstanceName= */ null, /* rootInstanceName= */ null, - CommonCertProviderTestUtils.getTestBootstrapInfo(), + CommonBootstrapperTestUtils.getTestBootstrapInfo(), /* alpnProtocols= */ null, /* staticCertValidationContext= */ null); fail("exception expected"); diff --git a/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderServerSslContextProviderTest.java b/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderServerSslContextProviderTest.java index 9de2081ed7c..ef801ccc2c1 100644 --- a/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderServerSslContextProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderServerSslContextProviderTest.java @@ -32,6 +32,7 @@ import io.envoyproxy.envoy.config.core.v3.DataSource; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CertificateValidationContext; import io.grpc.xds.Bootstrapper; +import io.grpc.xds.CommonBootstrapperTestUtils; import io.grpc.xds.EnvoyServerProtoData; import io.grpc.xds.internal.certprovider.CertProviderClientSslContextProviderTest.QueuedExecutor; import io.grpc.xds.internal.sds.CommonTlsContextTestsUtil; @@ -90,7 +91,7 @@ public void testProviderForServer_mtls() throws Exception { getSslContextProvider( "gcp_id", "gcp_id", - CommonCertProviderTestUtils.getTestBootstrapInfo(), + CommonBootstrapperTestUtils.getTestBootstrapInfo(), /* alpnProtocols= */ null, /* staticCertValidationContext= */ null, /* requireClientCert= */ true); @@ -154,7 +155,7 @@ public void testProviderForServer_queueExecutor() throws Exception { getSslContextProvider( "gcp_id", "gcp_id", - CommonCertProviderTestUtils.getTestBootstrapInfo(), + CommonBootstrapperTestUtils.getTestBootstrapInfo(), /* alpnProtocols= */ null, /* staticCertValidationContext= */ null, /* requireClientCert= */ true); @@ -188,7 +189,7 @@ public void testProviderForServer_tls() throws Exception { getSslContextProvider( "gcp_id", /* rootInstanceName= */ null, - CommonCertProviderTestUtils.getTestBootstrapInfo(), + CommonBootstrapperTestUtils.getTestBootstrapInfo(), /* alpnProtocols= */ null, /* staticCertValidationContext= */ null, /* requireClientCert= */ false); @@ -229,7 +230,7 @@ public void testProviderForServer_sslContextException_onError() throws Exception getSslContextProvider( /* certInstanceName= */ "gcp_id", /* rootInstanceName= */ "gcp_id", - CommonCertProviderTestUtils.getTestBootstrapInfo(), + CommonBootstrapperTestUtils.getTestBootstrapInfo(), /* alpnProtocols= */null, staticCertValidationContext, /* requireClientCert= */ true); @@ -266,7 +267,7 @@ public void testProviderForServer_certInstanceNull_expectError() throws Exceptio getSslContextProvider( /* certInstanceName= */ null, /* rootInstanceName= */ null, - CommonCertProviderTestUtils.getTestBootstrapInfo(), + CommonBootstrapperTestUtils.getTestBootstrapInfo(), /* alpnProtocols= */ null, /* staticCertValidationContext= */ null, /* requireClientCert= */ false); diff --git a/xds/src/test/java/io/grpc/xds/internal/certprovider/CommonCertProviderTestUtils.java b/xds/src/test/java/io/grpc/xds/internal/certprovider/CommonCertProviderTestUtils.java index 4e70164fe60..9507f3deac0 100644 --- a/xds/src/test/java/io/grpc/xds/internal/certprovider/CommonCertProviderTestUtils.java +++ b/xds/src/test/java/io/grpc/xds/internal/certprovider/CommonCertProviderTestUtils.java @@ -20,8 +20,6 @@ import com.google.common.io.CharStreams; import io.grpc.internal.testing.TestUtils; -import io.grpc.xds.Bootstrapper; -import io.grpc.xds.XdsInitializationException; import io.grpc.xds.internal.sds.trust.CertificateUtils; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; @@ -55,274 +53,6 @@ public class CommonCertProviderTestUtils { "-+END\\s+.*PRIVATE\\s+KEY[^-]*-+", // Footer Pattern.CASE_INSENSITIVE); - /** Creates a test bootstrap info object. */ - public static Bootstrapper.BootstrapInfo getTestBootstrapInfo() - throws XdsInitializationException { - String rawData = - "{\n" - + " \"xds_servers\": [],\n" - + " \"certificate_providers\": {\n" - + " \"gcp_id\": {\n" - + " \"plugin_name\": \"testca\",\n" - + " \"config\": {\n" - + " \"server\": {\n" - + " \"api_type\": \"GRPC\",\n" - + " \"grpc_services\": [{\n" - + " \"google_grpc\": {\n" - + " \"target_uri\": \"meshca.com\",\n" - + " \"channel_credentials\": {\"google_default\": {}},\n" - + " \"call_credentials\": [{\n" - + " \"sts_service\": {\n" - + " \"token_exchange_service\": \"securetoken.googleapis.com\",\n" - + " \"subject_token_path\": \"/etc/secret/sajwt.token\"\n" - + " }\n" - + " }]\n" // end call_credentials - + " },\n" // end google_grpc - + " \"time_out\": {\"seconds\": 10}\n" - + " }]\n" // end grpc_services - + " },\n" // end server - + " \"certificate_lifetime\": {\"seconds\": 86400},\n" - + " \"renewal_grace_period\": {\"seconds\": 3600},\n" - + " \"key_type\": \"RSA\",\n" - + " \"key_size\": 2048,\n" - + " \"location\": \"https://container.googleapis.com/v1/project/test-project1/locations/test-zone2/clusters/test-cluster3\"\n" - + " }\n" // end config - + " },\n" // end gcp_id - + " \"file_provider\": {\n" - + " \"plugin_name\": \"file_watcher\",\n" - + " \"config\": {\"path\": \"/etc/secret/certs\"}\n" - + " }\n" - + " }\n" - + "}"; - return Bootstrapper.parseConfig(rawData); - } - - static Bootstrapper.BootstrapInfo getNonDefaultTestBootstrapInfo() - throws XdsInitializationException { - String rawData = - "{\n" - + " \"xds_servers\": [],\n" - + " \"certificate_providers\": {\n" - + " \"gcp_id\": {\n" - + " \"plugin_name\": \"testca\",\n" - + " \"config\": {\n" - + " \"server\": {\n" - + " \"api_type\": \"GRPC\",\n" - + " \"grpc_services\": [{\n" - + " \"google_grpc\": {\n" - + " \"target_uri\": \"nonDefaultMeshCaUrl\",\n" - + " \"channel_credentials\": {\"google_default\": {}},\n" - + " \"call_credentials\": [{\n" - + " \"sts_service\": {\n" - + " \"token_exchange_service\": \"test.sts.com\",\n" - + " \"subject_token_path\": \"/tmp/path4\"\n" - + " }\n" - + " }]\n" // end call_credentials - + " },\n" // end google_grpc - + " \"time_out\": {\"seconds\": 12}\n" - + " }]\n" // end grpc_services - + " },\n" // end server - + " \"certificate_lifetime\": {\"seconds\": 234567},\n" - + " \"renewal_grace_period\": {\"seconds\": 4321},\n" - + " \"key_type\": \"RSA\",\n" - + " \"key_size\": 512,\n" - + " \"location\": \"https://container.googleapis.com/v1/projects/test-project1/locations/test-zone2/clusters/test-cluster3\"\n" - + " }\n" // end config - + " },\n" // end gcp_id - + " \"file_provider\": {\n" - + " \"plugin_name\": \"file_watcher\",\n" - + " \"config\": {\"path\": \"/etc/secret/certs\"}\n" - + " }\n" - + " }\n" - + "}"; - return Bootstrapper.parseConfig(rawData); - } - - static Bootstrapper.BootstrapInfo getMinimalBootstrapInfo() - throws XdsInitializationException { - String rawData = - "{\n" - + " \"xds_servers\": [],\n" - + " \"certificate_providers\": {\n" - + " \"gcp_id\": {\n" - + " \"plugin_name\": \"testca\",\n" - + " \"config\": {\n" - + " \"server\": {\n" - + " \"api_type\": \"GRPC\",\n" - + " \"grpc_services\": [{\n" - + " \"google_grpc\": {\n" - + " \"call_credentials\": [{\n" - + " \"sts_service\": {\n" - + " \"subject_token_path\": \"/tmp/path5\"\n" - + " }\n" - + " }]\n" // end call_credentials - + " }\n" // end google_grpc - + " }]\n" // end grpc_services - + " },\n" // end server - + " \"location\": \"https://container.googleapis.com/v1/projects/test-project1/locations/test-zone2/clusters/test-cluster3\"\n" - + " }\n" // end config - + " }\n" // end gcp_id - + " }\n" - + "}"; - return Bootstrapper.parseConfig(rawData); - } - - static Bootstrapper.BootstrapInfo getMinimalBootstrapInfo_v1beta1AndZone() - throws XdsInitializationException { - String rawData = - "{\n" - + " \"xds_servers\": [],\n" - + " \"certificate_providers\": {\n" - + " \"gcp_id\": {\n" - + " \"plugin_name\": \"testca\",\n" - + " \"config\": {\n" - + " \"server\": {\n" - + " \"api_type\": \"GRPC\",\n" - + " \"grpc_services\": [{\n" - + " \"google_grpc\": {\n" - + " \"call_credentials\": [{\n" - + " \"sts_service\": {\n" - + " \"subject_token_path\": \"/tmp/path5\"\n" - + " }\n" - + " }]\n" // end call_credentials - + " }\n" // end google_grpc - + " }]\n" // end grpc_services - + " },\n" // end server - + " \"location\": \"https://container.googleapis.com/v1beta1/projects/test-project1/zones/test-zone2/clusters/test-cluster3\"\n" - + " }\n" // end config - + " }\n" // end gcp_id - + " }\n" - + "}"; - return Bootstrapper.parseConfig(rawData); - } - - static Bootstrapper.BootstrapInfo getMinimalAndBadClusterUrlBootstrapInfo() - throws XdsInitializationException { - String rawData = - "{\n" - + " \"xds_servers\": [],\n" - + " \"certificate_providers\": {\n" - + " \"gcp_id\": {\n" - + " \"plugin_name\": \"testca\",\n" - + " \"config\": {\n" - + " \"server\": {\n" - + " \"api_type\": \"GRPC\",\n" - + " \"grpc_services\": [{\n" - + " \"google_grpc\": {\n" - + " \"call_credentials\": [{\n" - + " \"sts_service\": {\n" - + " \"subject_token_path\": \"/tmp/path5\"\n" - + " }\n" - + " }]\n" // end call_credentials - + " }\n" // end google_grpc - + " }]\n" // end grpc_services - + " },\n" // end server - + " \"location\": \"https://container.googleapis.com/v1/project/test-project1/locations/test-zone2/clusters/test-cluster3\"\n" - + " }\n" // end config - + " }\n" // end gcp_id - + " }\n" - + "}"; - return Bootstrapper.parseConfig(rawData); - } - - static Bootstrapper.BootstrapInfo getMissingSaJwtLocation() - throws XdsInitializationException { - String rawData = - "{\n" - + " \"xds_servers\": [],\n" - + " \"certificate_providers\": {\n" - + " \"gcp_id\": {\n" - + " \"plugin_name\": \"testca\",\n" - + " \"config\": {\n" - + " \"server\": {\n" - + " \"api_type\": \"GRPC\",\n" - + " \"grpc_services\": [{\n" - + " \"google_grpc\": {\n" - + " \"call_credentials\": [{\n" - + " \"sts_service\": {\n" - + " }\n" - + " }]\n" // end call_credentials - + " }\n" // end google_grpc - + " }]\n" // end grpc_services - + " },\n" // end server - + " \"location\": \"https://container.googleapis.com/v1/projects/test-project1/locations/test-zone2/clusters/test-cluster3\"\n" - + " }\n" // end config - + " }\n" // end gcp_id - + " }\n" - + "}"; - return Bootstrapper.parseConfig(rawData); - } - - static Bootstrapper.BootstrapInfo getMissingGkeClusterUrl() - throws XdsInitializationException { - String rawData = - "{\n" - + " \"xds_servers\": [],\n" - + " \"certificate_providers\": {\n" - + " \"gcp_id\": {\n" - + " \"plugin_name\": \"testca\",\n" - + " \"config\": {\n" - + " \"server\": {\n" - + " \"api_type\": \"GRPC\",\n" - + " \"grpc_services\": [{\n" - + " \"google_grpc\": {\n" - + " \"target_uri\": \"meshca.com\",\n" - + " \"channel_credentials\": {\"google_default\": {}},\n" - + " \"call_credentials\": [{\n" - + " \"sts_service\": {\n" - + " \"token_exchange_service\": \"securetoken.googleapis.com\",\n" - + " \"subject_token_path\": \"/etc/secret/sajwt.token\"\n" - + " }\n" - + " }]\n" // end call_credentials - + " },\n" // end google_grpc - + " \"time_out\": {\"seconds\": 10}\n" - + " }]\n" // end grpc_services - + " },\n" // end server - + " \"certificate_lifetime\": {\"seconds\": 86400},\n" - + " \"renewal_grace_period\": {\"seconds\": 3600},\n" - + " \"key_type\": \"RSA\",\n" - + " \"key_size\": 2048\n" - + " }\n" // end config - + " },\n" // end gcp_id - + " \"file_provider\": {\n" - + " \"plugin_name\": \"file_watcher\",\n" - + " \"config\": {\"path\": \"/etc/secret/certs\"}\n" - + " }\n" - + " }\n" - + "}"; - return Bootstrapper.parseConfig(rawData); - } - - static Bootstrapper.BootstrapInfo getBadChannelCredsConfig() - throws XdsInitializationException { - String rawData = - "{\n" - + " \"xds_servers\": [],\n" - + " \"certificate_providers\": {\n" - + " \"gcp_id\": {\n" - + " \"plugin_name\": \"testca\",\n" - + " \"config\": {\n" - + " \"server\": {\n" - + " \"api_type\": \"GRPC\",\n" - + " \"grpc_services\": [{\n" - + " \"google_grpc\": {\n" - + " \"channel_credentials\": {\"mtls\": \"true\"},\n" - + " \"call_credentials\": [{\n" - + " \"sts_service\": {\n" - + " \"subject_token_path\": \"/tmp/path5\"\n" - + " }\n" - + " }]\n" // end call_credentials - + " }\n" // end google_grpc - + " }]\n" // end grpc_services - + " },\n" // end server - + " \"location\": \"https://container.googleapis.com/v1/projects/test-project1/locations/test-zone2/clusters/test-cluster3\"\n" - + " }\n" // end config - + " }\n" // end gcp_id - + " }\n" - + "}"; - return Bootstrapper.parseConfig(rawData); - } - static PrivateKey getPrivateKey(String resourceName) throws Exception { InputStream inputStream = TestUtils.class.getResourceAsStream("/certs/" + resourceName); diff --git a/xds/src/test/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderProviderTest.java b/xds/src/test/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderProviderTest.java index d1bd116703b..791d5a395c5 100644 --- a/xds/src/test/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/certprovider/MeshCaCertificateProviderProviderTest.java @@ -29,9 +29,8 @@ import com.google.auth.oauth2.GoogleCredentials; import io.grpc.internal.BackoffPolicy; import io.grpc.internal.ExponentialBackoffPolicy; +import io.grpc.internal.JsonParser; import io.grpc.internal.TimeProvider; -import io.grpc.xds.Bootstrapper; -import io.grpc.xds.XdsInitializationException; import io.grpc.xds.internal.sts.StsCredentials; import java.io.IOException; import java.util.Map; @@ -107,10 +106,11 @@ public void providerRegisteredName() { } @Test - public void createProvider_minimalConfig() throws XdsInitializationException { + public void createProvider_minimalConfig() throws IOException { CertificateProvider.DistributorWatcher distWatcher = new CertificateProvider.DistributorWatcher(); - Map map = buildMinimalConfig(); + @SuppressWarnings("unchecked") + Map map = (Map) JsonParser.parse(MINIMAL_MESHCA_CONFIG); ScheduledExecutorService mockService = mock(ScheduledExecutorService.class); when(scheduledExecutorServiceFactory.create( eq(MeshCaCertificateProviderProvider.MESHCA_URL_DEFAULT))) @@ -143,10 +143,11 @@ public void createProvider_minimalConfig() throws XdsInitializationException { @Test public void createProvider_minimalConfig_v1beta1AndZone() - throws XdsInitializationException { + throws IOException { CertificateProvider.DistributorWatcher distWatcher = new CertificateProvider.DistributorWatcher(); - Map map = buildMinimalConfig_v1beta1AndZone(); + @SuppressWarnings("unchecked") + Map map = (Map) JsonParser.parse(V1BETA1_ZONE_MESHCA_CONFIG); ScheduledExecutorService mockService = mock(ScheduledExecutorService.class); when(scheduledExecutorServiceFactory.create( eq(MeshCaCertificateProviderProvider.MESHCA_URL_DEFAULT))) @@ -179,10 +180,11 @@ public void createProvider_minimalConfig_v1beta1AndZone() @Test public void createProvider_missingGkeUrl_expectException() - throws XdsInitializationException { + throws IOException { CertificateProvider.DistributorWatcher distWatcher = new CertificateProvider.DistributorWatcher(); - Map map = buildMissingGkeClusterUrlConfig(); + @SuppressWarnings("unchecked") + Map map = (Map) JsonParser.parse(MISSING_GKE_CLUSTER_URL_MESHCA_CONFIG); try { provider.createCertificateProvider(map, distWatcher, true); fail("exception expected"); @@ -192,11 +194,12 @@ public void createProvider_missingGkeUrl_expectException() } @Test - public void createProvider_missingGkeSaJwtLocation_expectException() - throws XdsInitializationException { + public void createProvider_missingSaJwtLocation_expectException() + throws IOException { CertificateProvider.DistributorWatcher distWatcher = new CertificateProvider.DistributorWatcher(); - Map map = buildMissingSaJwtLocationConfig(); + @SuppressWarnings("unchecked") + Map map = (Map) JsonParser.parse(MISSING_SAJWT_MESHCA_CONFIG); try { provider.createCertificateProvider(map, distWatcher, true); fail("exception expected"); @@ -207,10 +210,11 @@ public void createProvider_missingGkeSaJwtLocation_expectException() @Test public void createProvider_missingProject_expectException() - throws XdsInitializationException { + throws IOException { CertificateProvider.DistributorWatcher distWatcher = new CertificateProvider.DistributorWatcher(); - Map map = buildBadClusterUrlConfig(); + @SuppressWarnings("unchecked") + Map map = (Map) JsonParser.parse(MINIMAL_BAD_CLUSTER_URL_MESHCA_CONFIG); try { provider.createCertificateProvider(map, distWatcher, true); fail("exception expected"); @@ -221,10 +225,11 @@ public void createProvider_missingProject_expectException() @Test public void createProvider_badChannelCreds_expectException() - throws XdsInitializationException { + throws IOException { CertificateProvider.DistributorWatcher distWatcher = new CertificateProvider.DistributorWatcher(); - Map map = buildBadChannelCredsConfig(); + @SuppressWarnings("unchecked") + Map map = (Map) JsonParser.parse(BAD_CHANNEL_CREDS_MESHCA_CONFIG); try { provider.createCertificateProvider(map, distWatcher, true); fail("exception expected"); @@ -234,10 +239,11 @@ public void createProvider_badChannelCreds_expectException() } @Test - public void createProvider_nonDefaultFullConfig() throws XdsInitializationException { + public void createProvider_nonDefaultFullConfig() throws IOException { CertificateProvider.DistributorWatcher distWatcher = new CertificateProvider.DistributorWatcher(); - Map map = buildFullConfig(); + @SuppressWarnings("unchecked") + Map map = (Map) JsonParser.parse(NONDEFAULT_MESHCA_CONFIG); ScheduledExecutorService mockService = mock(ScheduledExecutorService.class); when(scheduledExecutorServiceFactory.create(eq(NON_DEFAULT_MESH_CA_URL))) .thenReturn(mockService); @@ -267,45 +273,137 @@ public void createProvider_nonDefaultFullConfig() throws XdsInitializationExcept eq(TimeUnit.SECONDS.toMillis(RPC_TIMEOUT_SECONDS))); } - private static Map buildFullConfig() throws XdsInitializationException { - return getCertProviderConfig(CommonCertProviderTestUtils.getNonDefaultTestBootstrapInfo()); - } + private static final String NONDEFAULT_MESHCA_CONFIG = + "{\n" + + " \"server\": {\n" + + " \"api_type\": \"GRPC\",\n" + + " \"grpc_services\": [{\n" + + " \"google_grpc\": {\n" + + " \"target_uri\": \"nonDefaultMeshCaUrl\",\n" + + " \"channel_credentials\": {\"google_default\": {}},\n" + + " \"call_credentials\": [{\n" + + " \"sts_service\": {\n" + + " \"token_exchange_service\": \"test.sts.com\",\n" + + " \"subject_token_path\": \"/tmp/path4\"\n" + + " }\n" + + " }]\n" // end call_credentials + + " },\n" // end google_grpc + + " \"time_out\": {\"seconds\": 12}\n" + + " }]\n" // end grpc_services + + " },\n" // end server + + " \"certificate_lifetime\": {\"seconds\": 234567},\n" + + " \"renewal_grace_period\": {\"seconds\": 4321},\n" + + " \"key_type\": \"RSA\",\n" + + " \"key_size\": 512,\n" + + " \"location\": \"https://container.googleapis.com/v1/projects/test-project1/locations/test-zone2/clusters/test-cluster3\"\n" + + " }"; - private static Map buildMinimalConfig() throws XdsInitializationException { - return getCertProviderConfig(CommonCertProviderTestUtils.getMinimalBootstrapInfo()); - } + private static final String MINIMAL_MESHCA_CONFIG = + "{\n" + + " \"server\": {\n" + + " \"api_type\": \"GRPC\",\n" + + " \"grpc_services\": [{\n" + + " \"google_grpc\": {\n" + + " \"call_credentials\": [{\n" + + " \"sts_service\": {\n" + + " \"subject_token_path\": \"/tmp/path5\"\n" + + " }\n" + + " }]\n" // end call_credentials + + " }\n" // end google_grpc + + " }]\n" // end grpc_services + + " },\n" // end server + + " \"location\": \"https://container.googleapis.com/v1/projects/test-project1/locations/test-zone2/clusters/test-cluster3\"\n" + + " }"; - private static Map buildMinimalConfig_v1beta1AndZone() - throws XdsInitializationException { - return getCertProviderConfig( - CommonCertProviderTestUtils.getMinimalBootstrapInfo_v1beta1AndZone()); - } + private static final String V1BETA1_ZONE_MESHCA_CONFIG = + "{\n" + + " \"server\": {\n" + + " \"api_type\": \"GRPC\",\n" + + " \"grpc_services\": [{\n" + + " \"google_grpc\": {\n" + + " \"call_credentials\": [{\n" + + " \"sts_service\": {\n" + + " \"subject_token_path\": \"/tmp/path5\"\n" + + " }\n" + + " }]\n" // end call_credentials + + " }\n" // end google_grpc + + " }]\n" // end grpc_services + + " },\n" // end server + + " \"location\": \"https://container.googleapis.com/v1beta1/projects/test-project1/zones/test-zone2/clusters/test-cluster3\"\n" + + " }"; - private static Map buildBadClusterUrlConfig() throws XdsInitializationException { - return getCertProviderConfig( - CommonCertProviderTestUtils.getMinimalAndBadClusterUrlBootstrapInfo()); - } + private static final String MINIMAL_BAD_CLUSTER_URL_MESHCA_CONFIG = + "{\n" + + " \"server\": {\n" + + " \"api_type\": \"GRPC\",\n" + + " \"grpc_services\": [{\n" + + " \"google_grpc\": {\n" + + " \"call_credentials\": [{\n" + + " \"sts_service\": {\n" + + " \"subject_token_path\": \"/tmp/path5\"\n" + + " }\n" + + " }]\n" // end call_credentials + + " }\n" // end google_grpc + + " }]\n" // end grpc_services + + " },\n" // end server + + " \"location\": \"https://container.googleapis.com/v1/project/test-project1/locations/test-zone2/clusters/test-cluster3\"\n" + + " }"; - private static Map buildMissingSaJwtLocationConfig() - throws XdsInitializationException { - return getCertProviderConfig(CommonCertProviderTestUtils.getMissingSaJwtLocation()); - } + private static final String MISSING_SAJWT_MESHCA_CONFIG = + "{\n" + + " \"server\": {\n" + + " \"api_type\": \"GRPC\",\n" + + " \"grpc_services\": [{\n" + + " \"google_grpc\": {\n" + + " \"call_credentials\": [{\n" + + " \"sts_service\": {\n" + + " }\n" + + " }]\n" // end call_credentials + + " }\n" // end google_grpc + + " }]\n" // end grpc_services + + " },\n" // end server + + " \"location\": \"https://container.googleapis.com/v1/projects/test-project1/locations/test-zone2/clusters/test-cluster3\"\n" + + " }"; - private static Map buildMissingGkeClusterUrlConfig() - throws XdsInitializationException { - return getCertProviderConfig(CommonCertProviderTestUtils.getMissingGkeClusterUrl()); - } - - private static Map buildBadChannelCredsConfig() - throws XdsInitializationException { - return getCertProviderConfig(CommonCertProviderTestUtils.getBadChannelCredsConfig()); - } + private static final String MISSING_GKE_CLUSTER_URL_MESHCA_CONFIG = + "{\n" + + " \"server\": {\n" + + " \"api_type\": \"GRPC\",\n" + + " \"grpc_services\": [{\n" + + " \"google_grpc\": {\n" + + " \"target_uri\": \"meshca.com\",\n" + + " \"channel_credentials\": {\"google_default\": {}},\n" + + " \"call_credentials\": [{\n" + + " \"sts_service\": {\n" + + " \"token_exchange_service\": \"securetoken.googleapis.com\",\n" + + " \"subject_token_path\": \"/etc/secret/sajwt.token\"\n" + + " }\n" + + " }]\n" // end call_credentials + + " },\n" // end google_grpc + + " \"time_out\": {\"seconds\": 10}\n" + + " }]\n" // end grpc_services + + " },\n" // end server + + " \"certificate_lifetime\": {\"seconds\": 86400},\n" + + " \"renewal_grace_period\": {\"seconds\": 3600},\n" + + " \"key_type\": \"RSA\",\n" + + " \"key_size\": 2048\n" + + " }"; - private static Map getCertProviderConfig(Bootstrapper.BootstrapInfo bootstrapInfo) { - Map certProviders = - bootstrapInfo.getCertProviders(); - Bootstrapper.CertificateProviderInfo gcpIdInfo = - certProviders.get("gcp_id"); - return gcpIdInfo.getConfig(); - } + private static final String BAD_CHANNEL_CREDS_MESHCA_CONFIG = + "{\n" + + " \"server\": {\n" + + " \"api_type\": \"GRPC\",\n" + + " \"grpc_services\": [{\n" + + " \"google_grpc\": {\n" + + " \"channel_credentials\": {\"mtls\": \"true\"},\n" + + " \"call_credentials\": [{\n" + + " \"sts_service\": {\n" + + " \"subject_token_path\": \"/tmp/path5\"\n" + + " }\n" + + " }]\n" // end call_credentials + + " }\n" // end google_grpc + + " }]\n" // end grpc_services + + " },\n" // end server + + " \"location\": \"https://container.googleapis.com/v1/projects/test-project1/locations/test-zone2/clusters/test-cluster3\"\n" + + " }"; } diff --git a/xds/src/test/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactoryTest.java b/xds/src/test/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactoryTest.java index 7b37c2d6449..46f0685b5a0 100644 --- a/xds/src/test/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactoryTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactoryTest.java @@ -30,6 +30,7 @@ import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext; import io.envoyproxy.envoy.type.matcher.v3.StringMatcher; import io.grpc.xds.Bootstrapper; +import io.grpc.xds.CommonBootstrapperTestUtils; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import io.grpc.xds.XdsInitializationException; import io.grpc.xds.internal.certprovider.CertProviderClientSslContextProvider; @@ -37,7 +38,6 @@ import io.grpc.xds.internal.certprovider.CertificateProviderProvider; import io.grpc.xds.internal.certprovider.CertificateProviderRegistry; import io.grpc.xds.internal.certprovider.CertificateProviderStore; -import io.grpc.xds.internal.certprovider.CommonCertProviderTestUtils; import io.grpc.xds.internal.certprovider.TestCertificateProvider; import java.io.IOException; import org.junit.Assert; @@ -131,7 +131,7 @@ public void createCertProviderClientSslContextProvider() throws XdsInitializatio /* alpnProtocols= */ null, /* staticCertValidationContext= */ null); - Bootstrapper.BootstrapInfo bootstrapInfo = CommonCertProviderTestUtils.getTestBootstrapInfo(); + Bootstrapper.BootstrapInfo bootstrapInfo = CommonBootstrapperTestUtils.getTestBootstrapInfo(); when(bootstrapper.readBootstrap()).thenReturn(bootstrapInfo); SslContextProvider sslContextProvider = clientSslContextProviderFactory.create(upstreamTlsContext); @@ -154,7 +154,7 @@ public void createCertProviderClientSslContextProvider_onlyRootCert() /* alpnProtocols= */ null, /* staticCertValidationContext= */ null); - Bootstrapper.BootstrapInfo bootstrapInfo = CommonCertProviderTestUtils.getTestBootstrapInfo(); + Bootstrapper.BootstrapInfo bootstrapInfo = CommonBootstrapperTestUtils.getTestBootstrapInfo(); when(bootstrapper.readBootstrap()).thenReturn(bootstrapInfo); SslContextProvider sslContextProvider = clientSslContextProviderFactory.create(upstreamTlsContext); @@ -184,7 +184,7 @@ public void createCertProviderClientSslContextProvider_withStaticContext() /* alpnProtocols= */ null, staticCertValidationContext); - Bootstrapper.BootstrapInfo bootstrapInfo = CommonCertProviderTestUtils.getTestBootstrapInfo(); + Bootstrapper.BootstrapInfo bootstrapInfo = CommonBootstrapperTestUtils.getTestBootstrapInfo(); when(bootstrapper.readBootstrap()).thenReturn(bootstrapInfo); SslContextProvider sslContextProvider = clientSslContextProviderFactory.create(upstreamTlsContext); @@ -211,7 +211,7 @@ public void createCertProviderClientSslContextProvider_2providers() /* alpnProtocols= */ null, /* staticCertValidationContext= */ null); - Bootstrapper.BootstrapInfo bootstrapInfo = CommonCertProviderTestUtils.getTestBootstrapInfo(); + Bootstrapper.BootstrapInfo bootstrapInfo = CommonBootstrapperTestUtils.getTestBootstrapInfo(); when(bootstrapper.readBootstrap()).thenReturn(bootstrapInfo); SslContextProvider sslContextProvider = clientSslContextProviderFactory.create(upstreamTlsContext); diff --git a/xds/src/test/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactoryTest.java b/xds/src/test/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactoryTest.java index 3f0f8cf2e04..00d32268b49 100644 --- a/xds/src/test/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactoryTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactoryTest.java @@ -30,13 +30,13 @@ import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext; import io.envoyproxy.envoy.type.matcher.v3.StringMatcher; import io.grpc.xds.Bootstrapper; +import io.grpc.xds.CommonBootstrapperTestUtils; import io.grpc.xds.EnvoyServerProtoData.DownstreamTlsContext; import io.grpc.xds.XdsInitializationException; import io.grpc.xds.internal.certprovider.CertProviderServerSslContextProvider; import io.grpc.xds.internal.certprovider.CertificateProvider; import io.grpc.xds.internal.certprovider.CertificateProviderRegistry; import io.grpc.xds.internal.certprovider.CertificateProviderStore; -import io.grpc.xds.internal.certprovider.CommonCertProviderTestUtils; import java.io.IOException; import org.junit.Assert; import org.junit.Before; @@ -128,7 +128,7 @@ public void createCertProviderServerSslContextProvider() throws XdsInitializatio /* staticCertValidationContext= */ null, /* requireClientCert= */ true); - Bootstrapper.BootstrapInfo bootstrapInfo = CommonCertProviderTestUtils.getTestBootstrapInfo(); + Bootstrapper.BootstrapInfo bootstrapInfo = CommonBootstrapperTestUtils.getTestBootstrapInfo(); when(bootstrapper.readBootstrap()).thenReturn(bootstrapInfo); SslContextProvider sslContextProvider = serverSslContextProviderFactory.create(downstreamTlsContext); @@ -152,7 +152,7 @@ public void createCertProviderServerSslContextProvider_onlyCertInstance() /* staticCertValidationContext= */ null, /* requireClientCert= */ true); - Bootstrapper.BootstrapInfo bootstrapInfo = CommonCertProviderTestUtils.getTestBootstrapInfo(); + Bootstrapper.BootstrapInfo bootstrapInfo = CommonBootstrapperTestUtils.getTestBootstrapInfo(); when(bootstrapper.readBootstrap()).thenReturn(bootstrapInfo); SslContextProvider sslContextProvider = serverSslContextProviderFactory.create(downstreamTlsContext); @@ -183,7 +183,7 @@ public void createCertProviderServerSslContextProvider_withStaticContext() staticCertValidationContext, /* requireClientCert= */ true); - Bootstrapper.BootstrapInfo bootstrapInfo = CommonCertProviderTestUtils.getTestBootstrapInfo(); + Bootstrapper.BootstrapInfo bootstrapInfo = CommonBootstrapperTestUtils.getTestBootstrapInfo(); when(bootstrapper.readBootstrap()).thenReturn(bootstrapInfo); SslContextProvider sslContextProvider = serverSslContextProviderFactory.create(downstreamTlsContext); @@ -211,7 +211,7 @@ public void createCertProviderServerSslContextProvider_2providers() /* staticCertValidationContext= */ null, /* requireClientCert= */ true); - Bootstrapper.BootstrapInfo bootstrapInfo = CommonCertProviderTestUtils.getTestBootstrapInfo(); + Bootstrapper.BootstrapInfo bootstrapInfo = CommonBootstrapperTestUtils.getTestBootstrapInfo(); when(bootstrapper.readBootstrap()).thenReturn(bootstrapInfo); SslContextProvider sslContextProvider = serverSslContextProviderFactory.create(downstreamTlsContext); From bf7a42dbd107b6d309518692dee7e950468cac15 Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Fri, 18 Sep 2020 12:54:48 -0700 Subject: [PATCH 51/86] api, core: delete io.grpc.LoadBalancer.loadBalancingConfig attribute (#7440) --- api/src/main/java/io/grpc/LoadBalancer.java | 12 ------------ .../internal/AutoConfiguredLoadBalancerFactory.java | 10 ---------- .../AutoConfiguredLoadBalancerFactoryTest.java | 7 ------- .../io/grpc/internal/ManagedChannelImplTest.java | 7 +++---- 4 files changed, 3 insertions(+), 33 deletions(-) diff --git a/api/src/main/java/io/grpc/LoadBalancer.java b/api/src/main/java/io/grpc/LoadBalancer.java index d2ed1ad9368..c609e2a54dc 100644 --- a/api/src/main/java/io/grpc/LoadBalancer.java +++ b/api/src/main/java/io/grpc/LoadBalancer.java @@ -110,18 +110,6 @@ @ExperimentalApi("https://github.com/grpc/grpc-java/issues/1771") @NotThreadSafe public abstract class LoadBalancer { - /** - * The load-balancing config converted from an JSON object injected by the GRPC library. - * - *

{@link NameResolver}s should not produce this attribute. - * - *

Deprecated: LB implementations should use parsed object from {@link - * LoadBalancerProvider#parseLoadBalancingPolicyConfig(Map)} instead of raw config. - */ - @Deprecated - @NameResolver.ResolutionResultAttr - public static final Attributes.Key> ATTR_LOAD_BALANCING_CONFIG = - Attributes.Key.create("io.grpc.LoadBalancer.loadBalancingConfig"); @Internal @NameResolver.ResolutionResultAttr diff --git a/core/src/main/java/io/grpc/internal/AutoConfiguredLoadBalancerFactory.java b/core/src/main/java/io/grpc/internal/AutoConfiguredLoadBalancerFactory.java index 5822c7b1914..24740d7bb36 100644 --- a/core/src/main/java/io/grpc/internal/AutoConfiguredLoadBalancerFactory.java +++ b/core/src/main/java/io/grpc/internal/AutoConfiguredLoadBalancerFactory.java @@ -17,7 +17,6 @@ package io.grpc.internal; import static com.google.common.base.Preconditions.checkNotNull; -import static io.grpc.LoadBalancer.ATTR_LOAD_BALANCING_CONFIG; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; @@ -111,11 +110,6 @@ public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { Status tryHandleResolvedAddresses(ResolvedAddresses resolvedAddresses) { List servers = resolvedAddresses.getAddresses(); Attributes attributes = resolvedAddresses.getAttributes(); - if (attributes.get(ATTR_LOAD_BALANCING_CONFIG) != null) { - throw new IllegalArgumentException( - "Unexpected ATTR_LOAD_BALANCING_CONFIG from upstream: " - + attributes.get(ATTR_LOAD_BALANCING_CONFIG)); - } PolicySelection policySelection = (PolicySelection) resolvedAddresses.getLoadBalancingPolicyConfig(); @@ -150,10 +144,6 @@ Status tryHandleResolvedAddresses(ResolvedAddresses resolvedAddresses) { if (lbConfig != null) { helper.getChannelLogger().log( ChannelLogLevel.DEBUG, "Load-balancing config: {0}", policySelection.config); - attributes = - attributes.toBuilder() - .set(ATTR_LOAD_BALANCING_CONFIG, policySelection.rawConfig) - .build(); } LoadBalancer delegate = getDelegate(); diff --git a/core/src/test/java/io/grpc/internal/AutoConfiguredLoadBalancerFactoryTest.java b/core/src/test/java/io/grpc/internal/AutoConfiguredLoadBalancerFactoryTest.java index 1f60bb97950..35a5fda02d1 100644 --- a/core/src/test/java/io/grpc/internal/AutoConfiguredLoadBalancerFactoryTest.java +++ b/core/src/test/java/io/grpc/internal/AutoConfiguredLoadBalancerFactoryTest.java @@ -17,7 +17,6 @@ package io.grpc.internal; import static com.google.common.truth.Truth.assertThat; -import static io.grpc.LoadBalancer.ATTR_LOAD_BALANCING_CONFIG; import static org.junit.Assert.assertTrue; import static org.mockito.AdditionalAnswers.delegatesTo; import static org.mockito.ArgumentMatchers.any; @@ -265,8 +264,6 @@ public void handleResolvedAddressGroups_propagateLbConfigToDelegate() throws Exc ArgumentCaptor.forClass(ResolvedAddresses.class); verify(testLbBalancer).handleResolvedAddresses(resultCaptor.capture()); assertThat(resultCaptor.getValue().getAddresses()).containsExactlyElementsIn(servers).inOrder(); - assertThat(resultCaptor.getValue().getAttributes().get(ATTR_LOAD_BALANCING_CONFIG)) - .containsExactly("setting1", "high"); verify(testLbBalancer, atLeast(0)).canHandleEmptyAddressListFromNameResolution(); ArgumentCaptor> lbConfigCaptor = ArgumentCaptor.forClass(Map.class); verify(testLbBalancerProvider).parseLoadBalancingPolicyConfig(lbConfigCaptor.capture()); @@ -288,8 +285,6 @@ public void handleResolvedAddressGroups_propagateLbConfigToDelegate() throws Exc verify(testLbBalancer, times(2)).handleResolvedAddresses(resultCaptor.capture()); assertThat(handleResult.getCode()).isEqualTo(Status.Code.OK); assertThat(resultCaptor.getValue().getAddresses()).containsExactlyElementsIn(servers).inOrder(); - assertThat(resultCaptor.getValue().getAttributes().get(ATTR_LOAD_BALANCING_CONFIG)) - .containsExactly("setting1", "low"); verify(testLbBalancerProvider, times(2)) .parseLoadBalancingPolicyConfig(lbConfigCaptor.capture()); assertThat(lbConfigCaptor.getValue()).containsExactly("setting1", "low"); @@ -383,8 +378,6 @@ public void handleResolvedAddressGroups_delegateAcceptsEmptyAddressList() assertThat(resultCaptor.getValue().getAddresses()).isEmpty(); assertThat(resultCaptor.getValue().getLoadBalancingPolicyConfig()) .isEqualTo(nextParsedConfigOrError2.get().getConfig()); - assertThat(resultCaptor.getValue().getAttributes().get(ATTR_LOAD_BALANCING_CONFIG)) - .containsExactly("setting1", "high"); } @Test diff --git a/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java b/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java index cd14f409ca3..5a7933e4bf2 100644 --- a/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java +++ b/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java @@ -1159,6 +1159,7 @@ public void nameResolverReturnsEmptySubLists_optionallyAllowed() throws Exceptio FakeNameResolverFactory nameResolverFactory = new FakeNameResolverFactory.Builder(expectedUri).build(); String rawLbConfig = "{ \"setting1\": \"high\" }"; + Object parsedLbConfig = new Object(); Map rawServiceConfig = parseConfig("{\"loadBalancingConfig\": [ {\"mock_lb\": " + rawLbConfig + " } ] }"); ManagedChannelServiceConfig parsedServiceConfig = @@ -1167,7 +1168,7 @@ public void nameResolverReturnsEmptySubLists_optionallyAllowed() throws Exceptio new PolicySelection( mockLoadBalancerProvider, parseConfig(rawLbConfig), - new Object())); + parsedLbConfig)); nameResolverFactory.nextConfigOrError.set(ConfigOrError.fromConfig(parsedServiceConfig)); channelBuilder.nameResolverFactory(nameResolverFactory); createChannel(); @@ -1178,9 +1179,7 @@ public void nameResolverReturnsEmptySubLists_optionallyAllowed() throws Exceptio ArgumentCaptor.forClass(ResolvedAddresses.class); verify(mockLoadBalancer).handleResolvedAddresses(resultCaptor.capture()); assertThat(resultCaptor.getValue().getAddresses()).isEmpty(); - Attributes actualAttrs = resultCaptor.getValue().getAttributes(); - Map lbConfig = actualAttrs.get(LoadBalancer.ATTR_LOAD_BALANCING_CONFIG); - assertEquals(ImmutableMap.of("setting1", "high"), lbConfig); + assertThat(resultCaptor.getValue().getLoadBalancingPolicyConfig()).isEqualTo(parsedLbConfig); // A no resolution retry assertEquals(0, timer.numPendingTasks(NAME_RESOLVER_REFRESH_TASK_FILTER)); From b571f23ad255884fc1bfebb1ab8eea4c849ea697 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Fri, 18 Sep 2020 16:31:08 -0400 Subject: [PATCH 52/86] core: Inline AbstractServerImplBuilder --- .../internal/AbstractServerImplBuilder.java | 330 ------------------ .../java/io/grpc/internal/ServerImpl.java | 2 +- .../io/grpc/internal/ServerImplBuilder.java | 272 +++++++++++++-- .../AbstractServerImplBuilderTest.java | 107 ------ .../grpc/internal/ServerImplBuilderTest.java | 86 +++-- .../java/io/grpc/internal/ServerImplTest.java | 23 +- 6 files changed, 321 insertions(+), 499 deletions(-) delete mode 100644 core/src/main/java/io/grpc/internal/AbstractServerImplBuilder.java delete mode 100644 core/src/test/java/io/grpc/internal/AbstractServerImplBuilderTest.java diff --git a/core/src/main/java/io/grpc/internal/AbstractServerImplBuilder.java b/core/src/main/java/io/grpc/internal/AbstractServerImplBuilder.java deleted file mode 100644 index c68076cc4c6..00000000000 --- a/core/src/main/java/io/grpc/internal/AbstractServerImplBuilder.java +++ /dev/null @@ -1,330 +0,0 @@ -/* - * Copyright 2014 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.internal; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.util.concurrent.MoreExecutors; -import io.grpc.BinaryLog; -import io.grpc.BindableService; -import io.grpc.CompressorRegistry; -import io.grpc.Context; -import io.grpc.Deadline; -import io.grpc.DecompressorRegistry; -import io.grpc.HandlerRegistry; -import io.grpc.InternalChannelz; -import io.grpc.Server; -import io.grpc.ServerBuilder; -import io.grpc.ServerInterceptor; -import io.grpc.ServerMethodDefinition; -import io.grpc.ServerServiceDefinition; -import io.grpc.ServerStreamTracer; -import io.grpc.ServerTransportFilter; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; -import javax.annotation.Nullable; - -/** - * The base class for server builders. - * - * @param The concrete type for this builder. - */ -public abstract class AbstractServerImplBuilder> - extends ServerBuilder { - - private static final Logger log = Logger.getLogger(AbstractServerImplBuilder.class.getName()); - - public static ServerBuilder forPort(int port) { - throw new UnsupportedOperationException("Subclass failed to hide static factory"); - } - - // defaults - private static final ObjectPool DEFAULT_EXECUTOR_POOL = - SharedResourcePool.forResource(GrpcUtil.SHARED_CHANNEL_EXECUTOR); - private static final HandlerRegistry DEFAULT_FALLBACK_REGISTRY = new DefaultFallbackRegistry(); - private static final DecompressorRegistry DEFAULT_DECOMPRESSOR_REGISTRY = - DecompressorRegistry.getDefaultInstance(); - private static final CompressorRegistry DEFAULT_COMPRESSOR_REGISTRY = - CompressorRegistry.getDefaultInstance(); - private static final long DEFAULT_HANDSHAKE_TIMEOUT_MILLIS = TimeUnit.SECONDS.toMillis(120); - - // mutable state - final InternalHandlerRegistry.Builder registryBuilder = - new InternalHandlerRegistry.Builder(); - final List transportFilters = new ArrayList<>(); - final List interceptors = new ArrayList<>(); - private final List streamTracerFactories = new ArrayList<>(); - HandlerRegistry fallbackRegistry = DEFAULT_FALLBACK_REGISTRY; - ObjectPool executorPool = DEFAULT_EXECUTOR_POOL; - DecompressorRegistry decompressorRegistry = DEFAULT_DECOMPRESSOR_REGISTRY; - CompressorRegistry compressorRegistry = DEFAULT_COMPRESSOR_REGISTRY; - long handshakeTimeoutMillis = DEFAULT_HANDSHAKE_TIMEOUT_MILLIS; - Deadline.Ticker ticker = Deadline.getSystemTicker(); - private boolean statsEnabled = true; - private boolean recordStartedRpcs = true; - private boolean recordFinishedRpcs = true; - private boolean recordRealTimeMetrics = false; - private boolean tracingEnabled = true; - @Nullable BinaryLog binlog; - TransportTracer.Factory transportTracerFactory = TransportTracer.getDefaultFactory(); - InternalChannelz channelz = InternalChannelz.instance(); - CallTracer.Factory callTracerFactory = CallTracer.getDefaultFactory(); - - @Override - public final T directExecutor() { - return executor(MoreExecutors.directExecutor()); - } - - @Override - public final T executor(@Nullable Executor executor) { - this.executorPool = executor != null ? new FixedObjectPool<>(executor) : DEFAULT_EXECUTOR_POOL; - return thisT(); - } - - @Override - public final T addService(ServerServiceDefinition service) { - registryBuilder.addService(checkNotNull(service, "service")); - return thisT(); - } - - @Override - public final T addService(BindableService bindableService) { - return addService(checkNotNull(bindableService, "bindableService").bindService()); - } - - @Override - public final T addTransportFilter(ServerTransportFilter filter) { - transportFilters.add(checkNotNull(filter, "filter")); - return thisT(); - } - - @Override - public final T intercept(ServerInterceptor interceptor) { - interceptors.add(checkNotNull(interceptor, "interceptor")); - return thisT(); - } - - @Override - public final T addStreamTracerFactory(ServerStreamTracer.Factory factory) { - streamTracerFactories.add(checkNotNull(factory, "factory")); - return thisT(); - } - - @Override - public final T fallbackHandlerRegistry(@Nullable HandlerRegistry registry) { - this.fallbackRegistry = registry != null ? registry : DEFAULT_FALLBACK_REGISTRY; - return thisT(); - } - - @Override - public final T decompressorRegistry(@Nullable DecompressorRegistry registry) { - this.decompressorRegistry = registry != null ? registry : DEFAULT_DECOMPRESSOR_REGISTRY; - return thisT(); - } - - @Override - public final T compressorRegistry(@Nullable CompressorRegistry registry) { - this.compressorRegistry = registry != null ? registry : DEFAULT_COMPRESSOR_REGISTRY; - return thisT(); - } - - @Override - public final T handshakeTimeout(long timeout, TimeUnit unit) { - checkArgument(timeout > 0, "handshake timeout is %s, but must be positive", timeout); - this.handshakeTimeoutMillis = checkNotNull(unit, "unit").toMillis(timeout); - return thisT(); - } - - @Override - public final T setBinaryLog(@Nullable BinaryLog binaryLog) { - this.binlog = binaryLog; - return thisT(); - } - - @VisibleForTesting - public final T setTransportTracerFactory(TransportTracer.Factory transportTracerFactory) { - this.transportTracerFactory = transportTracerFactory; - return thisT(); - } - - /** - * Disable or enable stats features. Enabled by default. - */ - protected void setStatsEnabled(boolean value) { - this.statsEnabled = value; - } - - /** - * Disable or enable stats recording for RPC upstarts. Effective only if {@link - * #setStatsEnabled} is set to true. Enabled by default. - */ - protected void setStatsRecordStartedRpcs(boolean value) { - recordStartedRpcs = value; - } - - /** - * Disable or enable stats recording for RPC completions. Effective only if {@link - * #setStatsEnabled} is set to true. Enabled by default. - */ - protected void setStatsRecordFinishedRpcs(boolean value) { - recordFinishedRpcs = value; - } - - /** - * Disable or enable real-time metrics recording. Effective only if {@link #setStatsEnabled} is - * set to true. Disabled by default. - */ - protected void setStatsRecordRealTimeMetrics(boolean value) { - recordRealTimeMetrics = value; - } - - /** - * Disable or enable tracing features. Enabled by default. - */ - protected void setTracingEnabled(boolean value) { - tracingEnabled = value; - } - - /** - * Sets a custom deadline ticker. This should only be called from InProcessServerBuilder. - */ - protected void setDeadlineTicker(Deadline.Ticker ticker) { - this.ticker = checkNotNull(ticker, "ticker"); - } - - @Override - public final Server build() { - return new ServerImpl(this, buildTransportServers(getTracerFactories()), Context.ROOT); - } - - @VisibleForTesting - final List getTracerFactories() { - ArrayList tracerFactories = new ArrayList<>(); - if (statsEnabled) { - ServerStreamTracer.Factory censusStatsTracerFactory = null; - try { - Class censusStatsAccessor = - Class.forName("io.grpc.census.InternalCensusStatsAccessor"); - Method getServerStreamTracerFactoryMethod = - censusStatsAccessor.getDeclaredMethod( - "getServerStreamTracerFactory", - boolean.class, - boolean.class, - boolean.class); - censusStatsTracerFactory = - (ServerStreamTracer.Factory) getServerStreamTracerFactoryMethod - .invoke( - null, - recordStartedRpcs, - recordFinishedRpcs, - recordRealTimeMetrics); - } catch (ClassNotFoundException e) { - // Replace these separate catch statements with multicatch when Android min-API >= 19 - log.log(Level.FINE, "Unable to apply census stats", e); - } catch (NoSuchMethodException e) { - log.log(Level.FINE, "Unable to apply census stats", e); - } catch (IllegalAccessException e) { - log.log(Level.FINE, "Unable to apply census stats", e); - } catch (InvocationTargetException e) { - log.log(Level.FINE, "Unable to apply census stats", e); - } - if (censusStatsTracerFactory != null) { - tracerFactories.add(censusStatsTracerFactory); - } - } - if (tracingEnabled) { - ServerStreamTracer.Factory tracingStreamTracerFactory = null; - try { - Class censusTracingAccessor = - Class.forName("io.grpc.census.InternalCensusTracingAccessor"); - Method getServerStreamTracerFactoryMethod = - censusTracingAccessor.getDeclaredMethod("getServerStreamTracerFactory"); - tracingStreamTracerFactory = - (ServerStreamTracer.Factory) getServerStreamTracerFactoryMethod.invoke(null); - } catch (ClassNotFoundException e) { - // Replace these separate catch statements with multicatch when Android min-API >= 19 - log.log(Level.FINE, "Unable to apply census stats", e); - } catch (NoSuchMethodException e) { - log.log(Level.FINE, "Unable to apply census stats", e); - } catch (IllegalAccessException e) { - log.log(Level.FINE, "Unable to apply census stats", e); - } catch (InvocationTargetException e) { - log.log(Level.FINE, "Unable to apply census stats", e); - } - if (tracingStreamTracerFactory != null) { - tracerFactories.add(tracingStreamTracerFactory); - } - } - tracerFactories.addAll(streamTracerFactories); - tracerFactories.trimToSize(); - return Collections.unmodifiableList(tracerFactories); - } - - protected InternalChannelz getChannelz() { - return channelz; - } - - protected final TransportTracer.Factory getTransportTracerFactory() { - return transportTracerFactory; - } - - /** - * Children of AbstractServerBuilder should override this method to provide transport specific - * information for the server. This method is mean for Transport implementors and should not be - * used by normal users. - * - * @param streamTracerFactories an immutable list of stream tracer factories - */ - protected abstract List buildTransportServers( - List streamTracerFactories); - - private T thisT() { - @SuppressWarnings("unchecked") - T thisT = (T) this; - return thisT; - } - - private static final class DefaultFallbackRegistry extends HandlerRegistry { - @Override - public List getServices() { - return Collections.emptyList(); - } - - @Nullable - @Override - public ServerMethodDefinition lookupMethod( - String methodName, @Nullable String authority) { - return null; - } - } - - /** - * Returns the internal ExecutorPool for offloading tasks. - */ - protected ObjectPool getExecutorPool() { - return this.executorPool; - } -} diff --git a/core/src/main/java/io/grpc/internal/ServerImpl.java b/core/src/main/java/io/grpc/internal/ServerImpl.java index ebeca770108..a400fca0338 100644 --- a/core/src/main/java/io/grpc/internal/ServerImpl.java +++ b/core/src/main/java/io/grpc/internal/ServerImpl.java @@ -135,7 +135,7 @@ public final class ServerImpl extends io.grpc.Server implements InternalInstrume * @param rootContext context that callbacks for new RPCs should be derived from */ ServerImpl( - AbstractServerImplBuilder builder, + ServerImplBuilder builder, List transportServers, Context rootContext) { this.executorPool = Preconditions.checkNotNull(builder.executorPool, "executorPool"); diff --git a/core/src/main/java/io/grpc/internal/ServerImplBuilder.java b/core/src/main/java/io/grpc/internal/ServerImplBuilder.java index ce92f1c65c3..9208394183d 100644 --- a/core/src/main/java/io/grpc/internal/ServerImplBuilder.java +++ b/core/src/main/java/io/grpc/internal/ServerImplBuilder.java @@ -16,20 +16,81 @@ package io.grpc.internal; -import com.google.common.base.Preconditions; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.util.concurrent.MoreExecutors; +import io.grpc.BinaryLog; +import io.grpc.BindableService; +import io.grpc.CompressorRegistry; +import io.grpc.Context; import io.grpc.Deadline; +import io.grpc.DecompressorRegistry; +import io.grpc.HandlerRegistry; import io.grpc.InternalChannelz; +import io.grpc.Server; import io.grpc.ServerBuilder; +import io.grpc.ServerInterceptor; +import io.grpc.ServerMethodDefinition; +import io.grpc.ServerServiceDefinition; import io.grpc.ServerStreamTracer; +import io.grpc.ServerTransportFilter; import java.io.File; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.annotation.Nullable; /** * Default builder for {@link io.grpc.Server} instances, for usage in Transport implementations. */ -public final class ServerImplBuilder extends AbstractServerImplBuilder { +public final class ServerImplBuilder extends ServerBuilder { + + private static final Logger log = Logger.getLogger(ServerImplBuilder.class.getName()); + + public static ServerBuilder forPort(int port) { + throw new UnsupportedOperationException( + "ClientTransportServersBuilder is required, use a constructor"); + } + + // defaults + private static final ObjectPool DEFAULT_EXECUTOR_POOL = + SharedResourcePool.forResource(GrpcUtil.SHARED_CHANNEL_EXECUTOR); + private static final HandlerRegistry DEFAULT_FALLBACK_REGISTRY = new DefaultFallbackRegistry(); + private static final DecompressorRegistry DEFAULT_DECOMPRESSOR_REGISTRY = + DecompressorRegistry.getDefaultInstance(); + private static final CompressorRegistry DEFAULT_COMPRESSOR_REGISTRY = + CompressorRegistry.getDefaultInstance(); + private static final long DEFAULT_HANDSHAKE_TIMEOUT_MILLIS = TimeUnit.SECONDS.toMillis(120); + + // mutable state + final InternalHandlerRegistry.Builder registryBuilder = + new InternalHandlerRegistry.Builder(); + final List transportFilters = new ArrayList<>(); + final List interceptors = new ArrayList<>(); + private final List streamTracerFactories = new ArrayList<>(); private final ClientTransportServersBuilder clientTransportServersBuilder; + HandlerRegistry fallbackRegistry = DEFAULT_FALLBACK_REGISTRY; + ObjectPool executorPool = DEFAULT_EXECUTOR_POOL; + DecompressorRegistry decompressorRegistry = DEFAULT_DECOMPRESSOR_REGISTRY; + CompressorRegistry compressorRegistry = DEFAULT_COMPRESSOR_REGISTRY; + long handshakeTimeoutMillis = DEFAULT_HANDSHAKE_TIMEOUT_MILLIS; + Deadline.Ticker ticker = Deadline.getSystemTicker(); + private boolean statsEnabled = true; + private boolean recordStartedRpcs = true; + private boolean recordFinishedRpcs = true; + private boolean recordRealTimeMetrics = false; + private boolean tracingEnabled = true; + @Nullable BinaryLog binlog; + InternalChannelz channelz = InternalChannelz.instance(); + CallTracer.Factory callTracerFactory = CallTracer.getDefaultFactory(); /** * An interface to provide to provide transport specific information for the server. This method @@ -44,62 +105,223 @@ List buildClientTransportServers( * Creates a new server builder with given transport servers provider. */ public ServerImplBuilder(ClientTransportServersBuilder clientTransportServersBuilder) { - this.clientTransportServersBuilder = Preconditions - .checkNotNull(clientTransportServersBuilder, "clientTransportServersBuilder"); + this.clientTransportServersBuilder = checkNotNull(clientTransportServersBuilder, + "clientTransportServersBuilder"); } @Override - protected List buildTransportServers( - List streamTracerFactories) { - return clientTransportServersBuilder.buildClientTransportServers(streamTracerFactories); + public ServerImplBuilder directExecutor() { + return executor(MoreExecutors.directExecutor()); } @Override - public void setDeadlineTicker(Deadline.Ticker ticker) { - super.setDeadlineTicker(ticker); + public ServerImplBuilder executor(@Nullable Executor executor) { + this.executorPool = executor != null ? new FixedObjectPool<>(executor) : DEFAULT_EXECUTOR_POOL; + return this; } @Override - public void setTracingEnabled(boolean value) { - super.setTracingEnabled(value); + public ServerImplBuilder addService(ServerServiceDefinition service) { + registryBuilder.addService(checkNotNull(service, "service")); + return this; } @Override - public void setStatsEnabled(boolean value) { - super.setStatsEnabled(value); + public ServerImplBuilder addService(BindableService bindableService) { + return addService(checkNotNull(bindableService, "bindableService").bindService()); } @Override - public void setStatsRecordStartedRpcs(boolean value) { - super.setStatsRecordStartedRpcs(value); + public ServerImplBuilder addTransportFilter(ServerTransportFilter filter) { + transportFilters.add(checkNotNull(filter, "filter")); + return this; } @Override - public void setStatsRecordFinishedRpcs(boolean value) { - super.setStatsRecordFinishedRpcs(value); + public ServerImplBuilder intercept(ServerInterceptor interceptor) { + interceptors.add(checkNotNull(interceptor, "interceptor")); + return this; + } + + @Override + public ServerImplBuilder addStreamTracerFactory(ServerStreamTracer.Factory factory) { + streamTracerFactories.add(checkNotNull(factory, "factory")); + return this; + } + + @Override + public ServerImplBuilder fallbackHandlerRegistry(@Nullable HandlerRegistry registry) { + this.fallbackRegistry = registry != null ? registry : DEFAULT_FALLBACK_REGISTRY; + return this; } @Override + public ServerImplBuilder decompressorRegistry(@Nullable DecompressorRegistry registry) { + this.decompressorRegistry = registry != null ? registry : DEFAULT_DECOMPRESSOR_REGISTRY; + return this; + } + + @Override + public ServerImplBuilder compressorRegistry(@Nullable CompressorRegistry registry) { + this.compressorRegistry = registry != null ? registry : DEFAULT_COMPRESSOR_REGISTRY; + return this; + } + + @Override + public ServerImplBuilder handshakeTimeout(long timeout, TimeUnit unit) { + checkArgument(timeout > 0, "handshake timeout is %s, but must be positive", timeout); + this.handshakeTimeoutMillis = checkNotNull(unit, "unit").toMillis(timeout); + return this; + } + + @Override + public ServerImplBuilder setBinaryLog(@Nullable BinaryLog binaryLog) { + this.binlog = binaryLog; + return this; + } + + /** + * Disable or enable stats features. Enabled by default. + */ + public void setStatsEnabled(boolean value) { + this.statsEnabled = value; + } + + /** + * Disable or enable stats recording for RPC upstarts. Effective only if {@link + * #setStatsEnabled} is set to true. Enabled by default. + */ + public void setStatsRecordStartedRpcs(boolean value) { + recordStartedRpcs = value; + } + + /** + * Disable or enable stats recording for RPC completions. Effective only if {@link + * #setStatsEnabled} is set to true. Enabled by default. + */ + public void setStatsRecordFinishedRpcs(boolean value) { + recordFinishedRpcs = value; + } + + /** + * Disable or enable real-time metrics recording. Effective only if {@link #setStatsEnabled} is + * set to true. Disabled by default. + */ public void setStatsRecordRealTimeMetrics(boolean value) { - super.setStatsRecordRealTimeMetrics(value); + recordRealTimeMetrics = value; + } + + /** + * Disable or enable tracing features. Enabled by default. + */ + public void setTracingEnabled(boolean value) { + tracingEnabled = value; + } + + /** + * Sets a custom deadline ticker. This should only be called from InProcessServerBuilder. + */ + public void setDeadlineTicker(Deadline.Ticker ticker) { + this.ticker = checkNotNull(ticker, "ticker"); } @Override + public Server build() { + return new ServerImpl(this, + clientTransportServersBuilder.buildClientTransportServers(getTracerFactories()), + Context.ROOT); + } + + @VisibleForTesting + List getTracerFactories() { + ArrayList tracerFactories = new ArrayList<>(); + if (statsEnabled) { + ServerStreamTracer.Factory censusStatsTracerFactory = null; + try { + Class censusStatsAccessor = + Class.forName("io.grpc.census.InternalCensusStatsAccessor"); + Method getServerStreamTracerFactoryMethod = + censusStatsAccessor.getDeclaredMethod( + "getServerStreamTracerFactory", + boolean.class, + boolean.class, + boolean.class); + censusStatsTracerFactory = + (ServerStreamTracer.Factory) getServerStreamTracerFactoryMethod + .invoke( + null, + recordStartedRpcs, + recordFinishedRpcs, + recordRealTimeMetrics); + } catch (ClassNotFoundException e) { + // Replace these separate catch statements with multicatch when Android min-API >= 19 + log.log(Level.FINE, "Unable to apply census stats", e); + } catch (NoSuchMethodException e) { + log.log(Level.FINE, "Unable to apply census stats", e); + } catch (IllegalAccessException e) { + log.log(Level.FINE, "Unable to apply census stats", e); + } catch (InvocationTargetException e) { + log.log(Level.FINE, "Unable to apply census stats", e); + } + if (censusStatsTracerFactory != null) { + tracerFactories.add(censusStatsTracerFactory); + } + } + if (tracingEnabled) { + ServerStreamTracer.Factory tracingStreamTracerFactory = null; + try { + Class censusTracingAccessor = + Class.forName("io.grpc.census.InternalCensusTracingAccessor"); + Method getServerStreamTracerFactoryMethod = + censusTracingAccessor.getDeclaredMethod("getServerStreamTracerFactory"); + tracingStreamTracerFactory = + (ServerStreamTracer.Factory) getServerStreamTracerFactoryMethod.invoke(null); + } catch (ClassNotFoundException e) { + // Replace these separate catch statements with multicatch when Android min-API >= 19 + log.log(Level.FINE, "Unable to apply census stats", e); + } catch (NoSuchMethodException e) { + log.log(Level.FINE, "Unable to apply census stats", e); + } catch (IllegalAccessException e) { + log.log(Level.FINE, "Unable to apply census stats", e); + } catch (InvocationTargetException e) { + log.log(Level.FINE, "Unable to apply census stats", e); + } + if (tracingStreamTracerFactory != null) { + tracerFactories.add(tracingStreamTracerFactory); + } + } + tracerFactories.addAll(streamTracerFactories); + tracerFactories.trimToSize(); + return Collections.unmodifiableList(tracerFactories); + } + public InternalChannelz getChannelz() { - return super.getChannelz(); + return channelz; } - @Override + private static final class DefaultFallbackRegistry extends HandlerRegistry { + @Override + public List getServices() { + return Collections.emptyList(); + } + + @Nullable + @Override + public ServerMethodDefinition lookupMethod( + String methodName, @Nullable String authority) { + return null; + } + } + + /** + * Returns the internal ExecutorPool for offloading tasks. + */ public ObjectPool getExecutorPool() { - return super.getExecutorPool(); + return this.executorPool; } @Override public ServerImplBuilder useTransportSecurity(File certChain, File privateKey) { throw new UnsupportedOperationException("TLS not supported in ServerImplBuilder"); } - - public static ServerBuilder forPort(int port) { - throw new UnsupportedOperationException("ClientTransportServersBuilder is required"); - } } diff --git a/core/src/test/java/io/grpc/internal/AbstractServerImplBuilderTest.java b/core/src/test/java/io/grpc/internal/AbstractServerImplBuilderTest.java deleted file mode 100644 index 77aced7826e..00000000000 --- a/core/src/test/java/io/grpc/internal/AbstractServerImplBuilderTest.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2017 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.internal; - -import static com.google.common.truth.Truth.assertThat; -import static org.junit.Assert.assertEquals; - -import io.grpc.Metadata; -import io.grpc.ServerStreamTracer; -import java.io.File; -import java.util.List; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Unit tests for {@link AbstractServerImplBuilder}. */ -@RunWith(JUnit4.class) -public class AbstractServerImplBuilderTest { - - private static final ServerStreamTracer.Factory DUMMY_USER_TRACER = - new ServerStreamTracer.Factory() { - @Override - public ServerStreamTracer newServerStreamTracer(String fullMethodName, Metadata headers) { - throw new UnsupportedOperationException(); - } - }; - - private Builder builder = new Builder(); - - @Test - public void getTracerFactories_default() { - builder.addStreamTracerFactory(DUMMY_USER_TRACER); - - List factories = builder.getTracerFactories(); - - assertEquals(3, factories.size()); - assertThat(factories.get(0).getClass().getName()) - .isEqualTo("io.grpc.census.CensusStatsModule$ServerTracerFactory"); - assertThat(factories.get(1).getClass().getName()) - .isEqualTo("io.grpc.census.CensusTracingModule$ServerTracerFactory"); - assertThat(factories.get(2)).isSameInstanceAs(DUMMY_USER_TRACER); - } - - @Test - public void getTracerFactories_disableStats() { - builder.addStreamTracerFactory(DUMMY_USER_TRACER); - builder.setStatsEnabled(false); - - List factories = builder.getTracerFactories(); - - assertEquals(2, factories.size()); - assertThat(factories.get(0).getClass().getName()) - .isEqualTo("io.grpc.census.CensusTracingModule$ServerTracerFactory"); - assertThat(factories.get(1)).isSameInstanceAs(DUMMY_USER_TRACER); - } - - @Test - public void getTracerFactories_disableTracing() { - builder.addStreamTracerFactory(DUMMY_USER_TRACER); - builder.setTracingEnabled(false); - - List factories = builder.getTracerFactories(); - - assertEquals(2, factories.size()); - assertThat(factories.get(0).getClass().getName()) - .isEqualTo("io.grpc.census.CensusStatsModule$ServerTracerFactory"); - assertThat(factories.get(1)).isSameInstanceAs(DUMMY_USER_TRACER); - } - - @Test - public void getTracerFactories_disableBoth() { - builder.addStreamTracerFactory(DUMMY_USER_TRACER); - builder.setTracingEnabled(false); - builder.setStatsEnabled(false); - List factories = builder.getTracerFactories(); - assertThat(factories).containsExactly(DUMMY_USER_TRACER); - } - - static class Builder extends AbstractServerImplBuilder { - - @Override - protected List buildTransportServers( - List streamTracerFactories) { - throw new UnsupportedOperationException(); - } - - @Override - public Builder useTransportSecurity(File certChain, File privateKey) { - throw new UnsupportedOperationException(); - } - } - -} diff --git a/core/src/test/java/io/grpc/internal/ServerImplBuilderTest.java b/core/src/test/java/io/grpc/internal/ServerImplBuilderTest.java index 3b5e3c7d8c3..1d6a2ec9f7f 100644 --- a/core/src/test/java/io/grpc/internal/ServerImplBuilderTest.java +++ b/core/src/test/java/io/grpc/internal/ServerImplBuilderTest.java @@ -16,49 +16,89 @@ package io.grpc.internal; +import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.verify; +import io.grpc.Metadata; import io.grpc.ServerStreamTracer; import io.grpc.internal.ServerImplBuilder.ClientTransportServersBuilder; import java.util.List; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -import org.mockito.ArgumentMatchers; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; /** Unit tests for {@link ServerImplBuilder}. */ @RunWith(JUnit4.class) public class ServerImplBuilderTest { - @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + private static final ServerStreamTracer.Factory DUMMY_USER_TRACER = + new ServerStreamTracer.Factory() { + @Override + public ServerStreamTracer newServerStreamTracer(String fullMethodName, Metadata headers) { + throw new UnsupportedOperationException(); + } + }; - @Mock private ClientTransportServersBuilder mockClientTransportServersBuilder; - @Mock private List mockServerStreamTracerFactories; - @Mock private List mockInternalServers; private ServerImplBuilder builder; @Before public void setUp() throws Exception { - builder = new ServerImplBuilder(mockClientTransportServersBuilder); + builder = new ServerImplBuilder( + new ClientTransportServersBuilder() { + @Override + public List buildClientTransportServers( + List streamTracerFactories) { + throw new UnsupportedOperationException(); + } + }); } @Test - public void buildTransportServers() { - doReturn(mockInternalServers).when(mockClientTransportServersBuilder) - .buildClientTransportServers(ArgumentMatchers.anyList()); - - List servers = builder - .buildTransportServers(mockServerStreamTracerFactories); - assertEquals(mockInternalServers, servers); - assertNotNull(servers); - verify(mockClientTransportServersBuilder) - .buildClientTransportServers(mockServerStreamTracerFactories); + public void getTracerFactories_default() { + builder.addStreamTracerFactory(DUMMY_USER_TRACER); + + List factories = builder.getTracerFactories(); + + assertEquals(3, factories.size()); + assertThat(factories.get(0).getClass().getName()) + .isEqualTo("io.grpc.census.CensusStatsModule$ServerTracerFactory"); + assertThat(factories.get(1).getClass().getName()) + .isEqualTo("io.grpc.census.CensusTracingModule$ServerTracerFactory"); + assertThat(factories.get(2)).isSameInstanceAs(DUMMY_USER_TRACER); + } + + @Test + public void getTracerFactories_disableStats() { + builder.addStreamTracerFactory(DUMMY_USER_TRACER); + builder.setStatsEnabled(false); + + List factories = builder.getTracerFactories(); + + assertEquals(2, factories.size()); + assertThat(factories.get(0).getClass().getName()) + .isEqualTo("io.grpc.census.CensusTracingModule$ServerTracerFactory"); + assertThat(factories.get(1)).isSameInstanceAs(DUMMY_USER_TRACER); + } + + @Test + public void getTracerFactories_disableTracing() { + builder.addStreamTracerFactory(DUMMY_USER_TRACER); + builder.setTracingEnabled(false); + + List factories = builder.getTracerFactories(); + + assertEquals(2, factories.size()); + assertThat(factories.get(0).getClass().getName()) + .isEqualTo("io.grpc.census.CensusStatsModule$ServerTracerFactory"); + assertThat(factories.get(1)).isSameInstanceAs(DUMMY_USER_TRACER); + } + + @Test + public void getTracerFactories_disableBoth() { + builder.addStreamTracerFactory(DUMMY_USER_TRACER); + builder.setTracingEnabled(false); + builder.setStatsEnabled(false); + List factories = builder.getTracerFactories(); + assertThat(factories).containsExactly(DUMMY_USER_TRACER); } } diff --git a/core/src/test/java/io/grpc/internal/ServerImplTest.java b/core/src/test/java/io/grpc/internal/ServerImplTest.java index b32833f3439..cf71a8b7afd 100644 --- a/core/src/test/java/io/grpc/internal/ServerImplTest.java +++ b/core/src/test/java/io/grpc/internal/ServerImplTest.java @@ -76,12 +76,12 @@ import io.grpc.Status; import io.grpc.StringMarshaller; import io.grpc.internal.ServerImpl.JumpToApplicationThreadServerStreamListener; +import io.grpc.internal.ServerImplBuilder.ClientTransportServersBuilder; import io.grpc.internal.testing.SingleMessageProducer; import io.grpc.internal.testing.TestServerStreamTracer; import io.grpc.util.MutableHandlerRegistry; import io.perfmark.PerfMark; import java.io.ByteArrayInputStream; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.InetSocketAddress; @@ -164,7 +164,7 @@ public Context filterContext(Context context) { }; @Mock private ObjectPool executorPool; - private Builder builder = new Builder(); + private ServerImplBuilder builder; private MutableHandlerRegistry mutableFallbackRegistry = new MutableHandlerRegistry(); private HandlerRegistry fallbackRegistry = mock( HandlerRegistry.class, @@ -201,6 +201,14 @@ public List getServices() { @Before public void startUp() throws IOException { MockitoAnnotations.initMocks(this); + builder = new ServerImplBuilder( + new ClientTransportServersBuilder() { + @Override + public List buildClientTransportServers( + List streamTracerFactories) { + throw new UnsupportedOperationException(); + } + }); builder.channelz = channelz; builder.ticker = timer.getDeadlineTicker(); streamTracerFactories = Arrays.asList(streamTracerFactory); @@ -1507,17 +1515,6 @@ public ListenableFuture getStats() { } } - private static class Builder extends AbstractServerImplBuilder { - @Override protected List buildTransportServers( - List streamTracerFactories) { - throw new UnsupportedOperationException(); - } - - @Override public Builder useTransportSecurity(File f1, File f2) { - throw new UnsupportedOperationException(); - } - } - /** Allows more precise catch blocks than plain Error to avoid catching AssertionError. */ private static final class TestError extends Error {} } From e6b61ea207f7f962a179e2dc61fad95d99685d1c Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Fri, 18 Sep 2020 16:37:55 -0700 Subject: [PATCH 53/86] xds: reimplement EDS LB policy with downstream LB config generations that migrate to hierarchical LB tree codepath (#7391) Implemented the new EDS LB policy, which generates a LB config for instantiating a hierarchical load balancing subtree. The subtree includes downstream LB policies: - priority LB policy, which load balances individual priorities separately - weighted-target LB policy, which load balances individual localities within the same priority separately - lrs LB policy, which applies load recording that aggregates load stats for each locality - leaf LB policy (round_robin) The EDS LB policy is the place that receives endpoint information from traffic director and organizes the data into a model for hierarchical load balancing for the cluster. --- .../java/io/grpc/xds/EdsLoadBalancer2.java | 434 +++++++++ .../io/grpc/xds/EdsLoadBalancerProvider.java | 66 +- .../xds/PriorityLoadBalancerProvider.java | 4 +- .../WeightedTargetLoadBalancerProvider.java | 1 - .../main/java/io/grpc/xds/XdsLbPolicies.java | 3 + .../services/io.grpc.LoadBalancerProvider | 2 + .../io/grpc/xds/EdsLoadBalancer2Test.java | 905 ++++++++++++++++++ .../grpc/xds/EdsLoadBalancerProviderTest.java | 47 +- 8 files changed, 1376 insertions(+), 86 deletions(-) create mode 100644 xds/src/main/java/io/grpc/xds/EdsLoadBalancer2.java create mode 100644 xds/src/test/java/io/grpc/xds/EdsLoadBalancer2Test.java diff --git a/xds/src/main/java/io/grpc/xds/EdsLoadBalancer2.java b/xds/src/main/java/io/grpc/xds/EdsLoadBalancer2.java new file mode 100644 index 00000000000..27304346034 --- /dev/null +++ b/xds/src/main/java/io/grpc/xds/EdsLoadBalancer2.java @@ -0,0 +1,434 @@ +/* + * Copyright 2019 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.xds; + +import static com.google.common.base.Preconditions.checkNotNull; +import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; +import static io.grpc.xds.XdsLbPolicies.LRS_POLICY_NAME; +import static io.grpc.xds.XdsLbPolicies.PRIORITY_POLICY_NAME; + +import com.google.common.annotations.VisibleForTesting; +import io.grpc.Attributes; +import io.grpc.ConnectivityState; +import io.grpc.EquivalentAddressGroup; +import io.grpc.InternalLogId; +import io.grpc.LoadBalancer; +import io.grpc.LoadBalancerProvider; +import io.grpc.LoadBalancerRegistry; +import io.grpc.Status; +import io.grpc.internal.ObjectPool; +import io.grpc.internal.ServiceConfigUtil.PolicySelection; +import io.grpc.util.ForwardingLoadBalancerHelper; +import io.grpc.util.GracefulSwitchLoadBalancer; +import io.grpc.xds.EdsLoadBalancerProvider.EdsConfig; +import io.grpc.xds.EnvoyProtoData.DropOverload; +import io.grpc.xds.EnvoyProtoData.LbEndpoint; +import io.grpc.xds.EnvoyProtoData.Locality; +import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints; +import io.grpc.xds.LoadStatsManager.LoadStatsStore; +import io.grpc.xds.LrsLoadBalancerProvider.LrsConfig; +import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig; +import io.grpc.xds.ThreadSafeRandom.ThreadSafeRandomImpl; +import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedPolicySelection; +import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedTargetConfig; +import io.grpc.xds.XdsClient.EndpointUpdate; +import io.grpc.xds.XdsClient.EndpointWatcher; +import io.grpc.xds.XdsLogger.XdsLogLevel; +import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import javax.annotation.Nullable; + +final class EdsLoadBalancer2 extends LoadBalancer { + private final XdsLogger logger; + private final LoadBalancerRegistry lbRegistry; + private final ThreadSafeRandom random; + private final GracefulSwitchLoadBalancer switchingLoadBalancer; + private ObjectPool xdsClientPool; + private XdsClient xdsClient; + private String cluster; + private EdsLbState edsLbState; + + EdsLoadBalancer2(LoadBalancer.Helper helper) { + this(helper, LoadBalancerRegistry.getDefaultRegistry(), ThreadSafeRandomImpl.instance); + } + + @VisibleForTesting + EdsLoadBalancer2( + LoadBalancer.Helper helper, LoadBalancerRegistry lbRegistry, ThreadSafeRandom random) { + this.lbRegistry = checkNotNull(lbRegistry, "lbRegistry"); + this.random = checkNotNull(random, "random"); + switchingLoadBalancer = new GracefulSwitchLoadBalancer(checkNotNull(helper, "helper")); + InternalLogId logId = InternalLogId.allocate("eds-lb", helper.getAuthority()); + logger = XdsLogger.withLogId(logId); + logger.log(XdsLogLevel.INFO, "Created"); + } + + @Override + public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { + logger.log(XdsLogLevel.DEBUG, "Received resolution result: {0}", resolvedAddresses); + if (xdsClientPool == null) { + xdsClientPool = resolvedAddresses.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL); + xdsClient = xdsClientPool.getObject(); + } + EdsConfig config = (EdsConfig) resolvedAddresses.getLoadBalancingPolicyConfig(); + if (logger.isLoggable(XdsLogLevel.INFO)) { + logger.log(XdsLogLevel.INFO, "Received EDS lb config: cluster={0}, " + + "eds_service_name={1}, endpoint_picking_policy={2}, report_load={3}", + config.clusterName, config.edsServiceName, + config.endpointPickingPolicy.getProvider().getPolicyName(), + config.lrsServerName != null); + } + if (cluster == null) { + cluster = config.clusterName; + } + if (edsLbState == null || !Objects.equals(edsLbState.edsServiceName, config.edsServiceName)) { + edsLbState = new EdsLbState(config.edsServiceName, config.lrsServerName); + switchingLoadBalancer.switchTo(edsLbState); + } + switchingLoadBalancer.handleResolvedAddresses(resolvedAddresses); + } + + @Override + public void handleNameResolutionError(Status error) { + logger.log(XdsLogLevel.WARNING, "Received name resolution error: {0}", error); + switchingLoadBalancer.handleNameResolutionError(error); + } + + @Override + public boolean canHandleEmptyAddressListFromNameResolution() { + return true; + } + + @Override + public void shutdown() { + logger.log(XdsLogLevel.INFO, "Shutdown"); + switchingLoadBalancer.shutdown(); + if (xdsClientPool != null) { + xdsClientPool.returnObject(xdsClient); + } + } + + private final class EdsLbState extends LoadBalancer.Factory { + @Nullable + private final String edsServiceName; + @Nullable + private final String lrsServerName; + private final String resourceName; + + private EdsLbState(@Nullable String edsServiceName, @Nullable String lrsServerName) { + this.edsServiceName = edsServiceName; + this.lrsServerName = lrsServerName; + resourceName = edsServiceName == null ? cluster : edsServiceName; + } + + @Override + public LoadBalancer newLoadBalancer(Helper helper) { + return new ChildLbState(helper); + } + + private final class ChildLbState extends LoadBalancer implements EndpointWatcher { + @Nullable + private final LoadStatsStore loadStatsStore; + private final DropHandlingLbHelper lbHelper; + private List endpointAddresses = Collections.emptyList(); + private Map> prioritizedLocalityWeights + = Collections.emptyMap(); + private ResolvedAddresses resolvedAddresses; + private PolicySelection localityPickingPolicy; + private PolicySelection endpointPickingPolicy; + @Nullable + private LoadBalancer lb; + + private ChildLbState(Helper helper) { + if (lrsServerName != null) { + loadStatsStore = xdsClient.addClientStats(cluster, edsServiceName); + xdsClient.reportClientStats(); + } else { + loadStatsStore = null; + } + lbHelper = new DropHandlingLbHelper(helper); + logger.log( + XdsLogLevel.INFO, + "Start endpoint watcher on {0} with xDS client {1}", resourceName, xdsClient); + xdsClient.watchEndpointData(resourceName, this); + } + + @Override + public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { + this.resolvedAddresses = resolvedAddresses; + EdsConfig config = (EdsConfig) resolvedAddresses.getLoadBalancingPolicyConfig(); + if (lb != null) { + if (!config.localityPickingPolicy.equals(localityPickingPolicy) + || !config.endpointPickingPolicy.equals(endpointPickingPolicy)) { + PriorityLbConfig childConfig = + generatePriorityLbConfig(cluster, edsServiceName, lrsServerName, + config.localityPickingPolicy, config.endpointPickingPolicy, lbRegistry, + prioritizedLocalityWeights); + // TODO(chengyuanzhang): to be deleted after migrating to use XdsClient API. + Attributes attributes; + if (lrsServerName != null) { + attributes = + resolvedAddresses.getAttributes().toBuilder() + .set(XdsAttributes.ATTR_CLUSTER_SERVICE_LOAD_STATS_STORE, loadStatsStore) + .build(); + } else { + attributes = resolvedAddresses.getAttributes(); + } + lb.handleResolvedAddresses( + resolvedAddresses.toBuilder() + .setAddresses(endpointAddresses) + .setAttributes(attributes) + .setLoadBalancingPolicyConfig(childConfig) + .build()); + } + } + localityPickingPolicy = config.localityPickingPolicy; + endpointPickingPolicy = config.endpointPickingPolicy; + } + + @Override + public void handleNameResolutionError(Status error) { + if (lb != null) { + lb.handleNameResolutionError(error); + } else { + lbHelper.helper.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(error)); + } + } + + @Override + public void shutdown() { + if (lrsServerName != null) { + xdsClient.cancelClientStatsReport(); + xdsClient.removeClientStats(cluster, edsServiceName); + } + xdsClient.cancelEndpointDataWatch(resourceName, this); + logger.log( + XdsLogLevel.INFO, + "Cancelled endpoint watcher on {0} with xDS client {1}", resourceName, xdsClient); + if (lb != null) { + lb.shutdown(); + } + } + + @Override + public boolean canHandleEmptyAddressListFromNameResolution() { + return true; + } + + @Override + public void onEndpointChanged(EndpointUpdate update) { + logger.log(XdsLogLevel.DEBUG, + "Received endpoint update from xDS client {0}: {1}", xdsClient, update); + if (logger.isLoggable(XdsLogLevel.INFO)) { + logger.log( + XdsLogLevel.INFO, + "Received endpoint update: cluster_name={0}, {1} localities, {2} drop categories", + update.getClusterName(), update.getLocalityLbEndpointsMap().size(), + update.getDropPolicies().size()); + } + lbHelper.updateDropPolicies(update.getDropPolicies()); + Map localityLbEndpoints = update.getLocalityLbEndpointsMap(); + endpointAddresses = new ArrayList<>(); + prioritizedLocalityWeights = new HashMap<>(); + for (Locality locality : localityLbEndpoints.keySet()) { + LocalityLbEndpoints localityLbInfo = localityLbEndpoints.get(locality); + int priority = localityLbInfo.getPriority(); + boolean discard = true; + for (LbEndpoint endpoint : localityLbInfo.getEndpoints()) { + if (endpoint.isHealthy()) { + discard = false; + EquivalentAddressGroup eag = + AddressFilter.setPathFilter( + endpoint.getAddress(), + Arrays.asList(priorityName(priority), localityName(locality))); + endpointAddresses.add(eag); + } + } + if (discard) { + logger.log(XdsLogLevel.INFO, "Discard locality {0} with 0 healthy endpoints"); + continue; + } + if (!prioritizedLocalityWeights.containsKey(priority)) { + prioritizedLocalityWeights.put(priority, new HashMap()); + } + prioritizedLocalityWeights.get(priority).put( + locality, localityLbInfo.getLocalityWeight()); + } + if (prioritizedLocalityWeights.isEmpty()) { + lbHelper.helper.updateBalancingState( + TRANSIENT_FAILURE, + new ErrorPicker( + Status.UNAVAILABLE.withDescription("No usable priority/locality/endpoint"))); + return; + } + if (lb == null) { + lb = lbRegistry.getProvider(PRIORITY_POLICY_NAME).newLoadBalancer(lbHelper); + } + if (localityPickingPolicy != null && endpointPickingPolicy != null) { + PriorityLbConfig config = generatePriorityLbConfig(cluster, edsServiceName, + lrsServerName, localityPickingPolicy, endpointPickingPolicy, lbRegistry, + prioritizedLocalityWeights); + // TODO(chengyuanzhang): to be deleted after migrating to use XdsClient API. + Attributes attributes; + if (lrsServerName != null) { + attributes = + resolvedAddresses.getAttributes().toBuilder() + .set(XdsAttributes.ATTR_CLUSTER_SERVICE_LOAD_STATS_STORE, loadStatsStore) + .build(); + } else { + attributes = resolvedAddresses.getAttributes(); + } + lb.handleResolvedAddresses( + resolvedAddresses.toBuilder() + .setAddresses(endpointAddresses) + .setAttributes(attributes) + .setLoadBalancingPolicyConfig(config) + .build()); + } + } + + @Override + public void onResourceDoesNotExist(String resourceName) { + logger.log(XdsLogLevel.INFO, "Resource {0} is unavailable", resourceName); + if (lb != null) { + lb.shutdown(); + lb = null; + } + lbHelper.helper.updateBalancingState( + TRANSIENT_FAILURE, + new ErrorPicker( + Status.UNAVAILABLE.withDescription( + "Resource " + resourceName + " is unavailable"))); + } + + @Override + public void onError(Status error) { + logger.log( + XdsLogLevel.WARNING, "Received error from xDS client {0}: {1}", xdsClient, error); + if (lb == null) { + lbHelper.helper.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(error)); + } + } + + private final class DropHandlingLbHelper extends ForwardingLoadBalancerHelper { + private final Helper helper; + private List dropPolicies = Collections.emptyList(); + + private DropHandlingLbHelper(Helper helper) { + this.helper = helper; + } + + @Override + public void updateBalancingState( + ConnectivityState newState, final SubchannelPicker newPicker) { + SubchannelPicker picker = new SubchannelPicker() { + List dropOverloads = dropPolicies; + + @Override + public PickResult pickSubchannel(PickSubchannelArgs args) { + for (DropOverload dropOverload : dropOverloads) { + int rand = random.nextInt(1_000_000); + if (rand < dropOverload.getDropsPerMillion()) { + logger.log( + XdsLogLevel.INFO, + "Drop request with category: {0}", dropOverload.getCategory()); + if (loadStatsStore != null) { + loadStatsStore.recordDroppedRequest(dropOverload.getCategory()); + } + return PickResult.withDrop( + Status.UNAVAILABLE.withDescription("Dropped: " + dropOverload.getCategory())); + } + } + return newPicker.pickSubchannel(args); + } + }; + delegate().updateBalancingState(newState, picker); + } + + @Override + protected Helper delegate() { + return helper; + } + + private void updateDropPolicies(List dropOverloads) { + dropPolicies = dropOverloads; + } + } + } + } + + @VisibleForTesting + static PriorityLbConfig generatePriorityLbConfig( + String cluster, String edsServiceName, String lrsServerName, + PolicySelection localityPickingPolicy, PolicySelection endpointPickingPolicy, + LoadBalancerRegistry lbRegistry, + Map> prioritizedLocalityWeights) { + Map childPolicies = new HashMap<>(); + List priorities = new ArrayList<>(); + for (Integer priority : prioritizedLocalityWeights.keySet()) { + WeightedTargetConfig childConfig = + generateWeightedTargetLbConfig(cluster, edsServiceName, lrsServerName, + endpointPickingPolicy, lbRegistry, prioritizedLocalityWeights.get(priority)); + PolicySelection childPolicySelection = + new PolicySelection(localityPickingPolicy.getProvider(), null, childConfig); + String childName = priorityName(priority); + childPolicies.put(childName, childPolicySelection); + priorities.add(childName); + } + Collections.sort(priorities); + return new PriorityLbConfig( + Collections.unmodifiableMap(childPolicies), Collections.unmodifiableList(priorities)); + } + + @VisibleForTesting + static WeightedTargetConfig generateWeightedTargetLbConfig( + String cluster, String edsServiceName, String lrsServerName, + PolicySelection endpointPickingPolicy, LoadBalancerRegistry lbRegistry, + Map localityWeights) { + Map targets = new HashMap<>(); + for (Locality locality : localityWeights.keySet()) { + int weight = localityWeights.get(locality); + PolicySelection childPolicy; + if (lrsServerName != null) { + LrsConfig childConfig = + new LrsConfig(cluster, edsServiceName, lrsServerName, locality, endpointPickingPolicy); + LoadBalancerProvider childPolicyProvider = lbRegistry.getProvider(LRS_POLICY_NAME); + childPolicy = new PolicySelection(childPolicyProvider, null, childConfig); + } else { + childPolicy = endpointPickingPolicy; + } + targets.put(localityName(locality), new WeightedPolicySelection(weight, childPolicy)); + } + return new WeightedTargetConfig(Collections.unmodifiableMap(targets)); + } + + /** Generate a string to be used as the key for the given priority in the LB policy config. */ + private static String priorityName(int priority) { + return "priority" + priority; + } + + /** Generate a string to be used as the key for the given locality in the LB policy config. */ + private static String localityName(Locality locality) { + return locality.toString(); + } +} diff --git a/xds/src/main/java/io/grpc/xds/EdsLoadBalancerProvider.java b/xds/src/main/java/io/grpc/xds/EdsLoadBalancerProvider.java index 87ac8f76976..2a8acc8afab 100644 --- a/xds/src/main/java/io/grpc/xds/EdsLoadBalancerProvider.java +++ b/xds/src/main/java/io/grpc/xds/EdsLoadBalancerProvider.java @@ -17,23 +17,17 @@ package io.grpc.xds; import static com.google.common.base.Preconditions.checkNotNull; +import static io.grpc.xds.XdsLbPolicies.WEIGHTED_TARGET_POLICY_NAME; import com.google.common.base.MoreObjects; import com.google.common.base.Objects; -import com.google.common.collect.ImmutableMap; import io.grpc.Internal; import io.grpc.LoadBalancer; import io.grpc.LoadBalancer.Helper; import io.grpc.LoadBalancerProvider; import io.grpc.LoadBalancerRegistry; import io.grpc.NameResolver.ConfigOrError; -import io.grpc.Status; -import io.grpc.internal.JsonUtil; -import io.grpc.internal.ServiceConfigUtil; -import io.grpc.internal.ServiceConfigUtil.LbConfig; import io.grpc.internal.ServiceConfigUtil.PolicySelection; -import java.util.Collections; -import java.util.List; import java.util.Map; import javax.annotation.Nullable; @@ -68,60 +62,43 @@ public LoadBalancer newLoadBalancer(Helper helper) { @Override public ConfigOrError parseLoadBalancingPolicyConfig( Map rawLoadBalancingPolicyConfig) { - LoadBalancerRegistry registry = LoadBalancerRegistry.getDefaultRegistry(); - try { - String cluster = JsonUtil.getString(rawLoadBalancingPolicyConfig, "cluster"); - if (cluster == null) { - return ConfigOrError.fromError(Status.INTERNAL.withDescription("Cluster name required")); - } - String edsServiceName = JsonUtil.getString(rawLoadBalancingPolicyConfig, "edsServiceName"); - String lrsServerName = - JsonUtil.getString(rawLoadBalancingPolicyConfig, "lrsLoadReportingServerName"); - - // TODO(chengyuanzhang): figure out locality_picking_policy parsing and its default value. - - LbConfig roundRobinConfig = new LbConfig("round_robin", ImmutableMap.of()); - List endpointPickingPolicy = - ServiceConfigUtil - .unwrapLoadBalancingConfigList( - JsonUtil.getListOfObjects( - rawLoadBalancingPolicyConfig, "endpointPickingPolicy")); - if (endpointPickingPolicy == null || endpointPickingPolicy.isEmpty()) { - endpointPickingPolicy = Collections.singletonList(roundRobinConfig); - } - ConfigOrError endpointPickingConfigOrError = - ServiceConfigUtil.selectLbPolicyFromList(endpointPickingPolicy, registry); - if (endpointPickingConfigOrError.getError() != null) { - return endpointPickingConfigOrError; - } - PolicySelection endpointPickingSelection = - (PolicySelection) endpointPickingConfigOrError.getConfig(); - return ConfigOrError.fromConfig( - new EdsConfig(cluster, edsServiceName, lrsServerName, endpointPickingSelection)); - } catch (RuntimeException e) { - return ConfigOrError.fromError( - Status.fromThrowable(e).withDescription( - "Failed to parse EDS LB config: " + rawLoadBalancingPolicyConfig)); - } + throw new UnsupportedOperationException("not supported as top-level LB policy"); } static final class EdsConfig { - final String clusterName; @Nullable final String edsServiceName; @Nullable final String lrsServerName; + final PolicySelection localityPickingPolicy; final PolicySelection endpointPickingPolicy; + // TODO(chengyuanzhang): delete me. + EdsConfig( + String clusterName, + @Nullable String edsServiceName, + @Nullable String lrsServerName, + PolicySelection endpointPickingPolicy) { + this.clusterName = checkNotNull(clusterName, "clusterName"); + this.edsServiceName = edsServiceName; + this.lrsServerName = lrsServerName; + this.endpointPickingPolicy = checkNotNull(endpointPickingPolicy, "endpointPickingPolicy"); + LoadBalancerProvider provider = + LoadBalancerRegistry.getDefaultRegistry().getProvider(WEIGHTED_TARGET_POLICY_NAME); + localityPickingPolicy = new PolicySelection(provider, null, null); + } + EdsConfig( String clusterName, @Nullable String edsServiceName, @Nullable String lrsServerName, + PolicySelection localityPickingPolicy, PolicySelection endpointPickingPolicy) { this.clusterName = checkNotNull(clusterName, "clusterName"); this.edsServiceName = edsServiceName; this.lrsServerName = lrsServerName; + this.localityPickingPolicy = checkNotNull(localityPickingPolicy, "localityPickingPolicy"); this.endpointPickingPolicy = checkNotNull(endpointPickingPolicy, "endpointPickingPolicy"); } @@ -131,6 +108,7 @@ public String toString() { .add("clusterName", clusterName) .add("edsServiceName", edsServiceName) .add("lrsServerName", lrsServerName) + .add("localityPickingPolicy", localityPickingPolicy) .add("endpointPickingPolicy", endpointPickingPolicy) .toString(); } @@ -144,6 +122,7 @@ public boolean equals(Object obj) { return Objects.equal(this.clusterName, that.clusterName) && Objects.equal(this.edsServiceName, that.edsServiceName) && Objects.equal(this.lrsServerName, that.lrsServerName) + && Objects.equal(this.localityPickingPolicy, that.localityPickingPolicy) && Objects.equal(this.endpointPickingPolicy, that.endpointPickingPolicy); } @@ -154,6 +133,7 @@ public int hashCode() { clusterName, edsServiceName, lrsServerName, + localityPickingPolicy, endpointPickingPolicy); } } diff --git a/xds/src/main/java/io/grpc/xds/PriorityLoadBalancerProvider.java b/xds/src/main/java/io/grpc/xds/PriorityLoadBalancerProvider.java index 76afe210c99..88297108de7 100644 --- a/xds/src/main/java/io/grpc/xds/PriorityLoadBalancerProvider.java +++ b/xds/src/main/java/io/grpc/xds/PriorityLoadBalancerProvider.java @@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import com.google.common.base.MoreObjects; +import io.grpc.Internal; import io.grpc.LoadBalancer; import io.grpc.LoadBalancer.Helper; import io.grpc.LoadBalancerProvider; @@ -31,7 +32,8 @@ import java.util.Map; /** Provider for priority load balancing policy. */ -final class PriorityLoadBalancerProvider extends LoadBalancerProvider { +@Internal +public final class PriorityLoadBalancerProvider extends LoadBalancerProvider { @Override public boolean isAvailable() { diff --git a/xds/src/main/java/io/grpc/xds/WeightedTargetLoadBalancerProvider.java b/xds/src/main/java/io/grpc/xds/WeightedTargetLoadBalancerProvider.java index 78ed6fe3f8c..15a56458df5 100644 --- a/xds/src/main/java/io/grpc/xds/WeightedTargetLoadBalancerProvider.java +++ b/xds/src/main/java/io/grpc/xds/WeightedTargetLoadBalancerProvider.java @@ -127,7 +127,6 @@ static final class WeightedPolicySelection { final int weight; final PolicySelection policySelection; - @VisibleForTesting WeightedPolicySelection(int weight, PolicySelection policySelection) { this.weight = weight; this.policySelection = policySelection; diff --git a/xds/src/main/java/io/grpc/xds/XdsLbPolicies.java b/xds/src/main/java/io/grpc/xds/XdsLbPolicies.java index 06ec45dc915..f0ac6426ee0 100644 --- a/xds/src/main/java/io/grpc/xds/XdsLbPolicies.java +++ b/xds/src/main/java/io/grpc/xds/XdsLbPolicies.java @@ -20,7 +20,10 @@ final class XdsLbPolicies { static final String CLUSTER_MANAGER_POLICY_NAME = "cluster_manager_experimental"; static final String CDS_POLICY_NAME = "cds_experimental"; static final String EDS_POLICY_NAME = "eds_experimental"; + static final String PRIORITY_POLICY_NAME = "priority_experimental"; static final String WEIGHTED_TARGET_POLICY_NAME = "weighted_target_experimental"; + static final String LRS_POLICY_NAME = "lrs_experimental"; + // TODO(chengyuanzhang): delete routing policy. static final String XDS_ROUTING_POLICY_NAME = "xds_routing_experimental"; private XdsLbPolicies() {} diff --git a/xds/src/main/resources/META-INF/services/io.grpc.LoadBalancerProvider b/xds/src/main/resources/META-INF/services/io.grpc.LoadBalancerProvider index a3baf7399e2..ff774497bb4 100644 --- a/xds/src/main/resources/META-INF/services/io.grpc.LoadBalancerProvider +++ b/xds/src/main/resources/META-INF/services/io.grpc.LoadBalancerProvider @@ -1,5 +1,7 @@ io.grpc.xds.CdsLoadBalancerProvider io.grpc.xds.EdsLoadBalancerProvider +io.grpc.xds.PriorityLoadBalancerProvider io.grpc.xds.WeightedTargetLoadBalancerProvider +io.grpc.xds.LrsLoadBalancerProvider io.grpc.xds.XdsRoutingLoadBalancerProvider io.grpc.xds.ClusterManagerLoadBalancerProvider diff --git a/xds/src/test/java/io/grpc/xds/EdsLoadBalancer2Test.java b/xds/src/test/java/io/grpc/xds/EdsLoadBalancer2Test.java new file mode 100644 index 00000000000..5b39f255a6f --- /dev/null +++ b/xds/src/test/java/io/grpc/xds/EdsLoadBalancer2Test.java @@ -0,0 +1,905 @@ +/* + * Copyright 2019 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.xds; + +import static com.google.common.truth.Truth.assertThat; +import static io.grpc.xds.XdsLbPolicies.LRS_POLICY_NAME; +import static io.grpc.xds.XdsLbPolicies.PRIORITY_POLICY_NAME; +import static io.grpc.xds.XdsLbPolicies.WEIGHTED_TARGET_POLICY_NAME; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import io.grpc.Attributes; +import io.grpc.ConnectivityState; +import io.grpc.EquivalentAddressGroup; +import io.grpc.LoadBalancer; +import io.grpc.LoadBalancer.CreateSubchannelArgs; +import io.grpc.LoadBalancer.Helper; +import io.grpc.LoadBalancer.PickResult; +import io.grpc.LoadBalancer.PickSubchannelArgs; +import io.grpc.LoadBalancer.ResolvedAddresses; +import io.grpc.LoadBalancer.Subchannel; +import io.grpc.LoadBalancer.SubchannelPicker; +import io.grpc.LoadBalancerProvider; +import io.grpc.LoadBalancerRegistry; +import io.grpc.ManagedChannel; +import io.grpc.NameResolver; +import io.grpc.Status; +import io.grpc.Status.Code; +import io.grpc.SynchronizationContext; +import io.grpc.internal.FakeClock; +import io.grpc.internal.ObjectPool; +import io.grpc.internal.ServiceConfigUtil.PolicySelection; +import io.grpc.xds.EdsLoadBalancerProvider.EdsConfig; +import io.grpc.xds.EnvoyProtoData.DropOverload; +import io.grpc.xds.EnvoyProtoData.LbEndpoint; +import io.grpc.xds.EnvoyProtoData.Locality; +import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints; +import io.grpc.xds.LoadStatsManager.LoadStatsStore; +import io.grpc.xds.LrsLoadBalancerProvider.LrsConfig; +import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig; +import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedPolicySelection; +import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedTargetConfig; +import java.net.SocketAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.AtomicLong; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +/** + * Tests for {@link EdsLoadBalancer2}. + */ +@RunWith(JUnit4.class) +public class EdsLoadBalancer2Test { + private static final String CLUSTER = "cluster-foo.googleapis.com"; + private static final String AUTHORITY = "api.google.com"; + private static final String EDS_SERVICE_NAME = "service.googleapis.com"; + private static final String LRS_SERVER_NAME = "lrs.googleapis.com"; + private final Locality locality1 = + new Locality("test-region-1", "test-zone-1", "test-subzone-1"); + private final Locality locality2 = + new Locality("test-region-2", "test-zone-2", "test-subzone-2"); + private final Locality locality3 = + new Locality("test-region-3", "test-zone-3", "test-subzone-3"); + private final SynchronizationContext syncContext = new SynchronizationContext( + new Thread.UncaughtExceptionHandler() { + @Override + public void uncaughtException(Thread t, Throwable e) { + throw new AssertionError(e); + } + }); + + private final FakeClock fakeClock = new FakeClock(); + private final LoadBalancerRegistry registry = new LoadBalancerRegistry(); + private final PolicySelection roundRobin = + new PolicySelection(new FakeLoadBalancerProvider("round_robin"), null, null); + private final PolicySelection weightedTarget = + new PolicySelection(new FakeLoadBalancerProvider(WEIGHTED_TARGET_POLICY_NAME), null, null); + private final List downstreamBalancers = new ArrayList<>(); + private final FakeXdsClient xdsClient = new FakeXdsClient(); + private final ObjectPool xdsClientPool = new ObjectPool() { + @Override + public XdsClient getObject() { + xdsClientRefs++; + return xdsClient; + } + + @Override + public XdsClient returnObject(Object object) { + xdsClientRefs--; + return null; + } + }; + private LoadBalancer.Helper helper = new FakeLbHelper(); + @Mock + private ThreadSafeRandom mockRandom; + private int xdsClientRefs; + private ConnectivityState currentState; + private SubchannelPicker currentPicker; + private EdsLoadBalancer2 loadBalancer; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + + registry.register(new FakeLoadBalancerProvider(PRIORITY_POLICY_NAME)); + registry.register(new FakeLoadBalancerProvider(LRS_POLICY_NAME)); + loadBalancer = new EdsLoadBalancer2(helper, registry, mockRandom); + loadBalancer.handleResolvedAddresses( + ResolvedAddresses.newBuilder() + .setAddresses(Collections.emptyList()) + .setAttributes( + Attributes.newBuilder().set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool).build()) + .setLoadBalancingPolicyConfig( + new EdsConfig( + CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_NAME, weightedTarget, roundRobin)) + .build()); + } + + @After + public void tearDown() { + loadBalancer.shutdown(); + assertThat(xdsClient.watchers).isEmpty(); + assertThat(xdsClient.dropStats).isEmpty(); + assertThat(xdsClientRefs).isEqualTo(0); + assertThat(downstreamBalancers).isEmpty(); + } + + @Test + public void receiveFirstEndpointResource() { + EquivalentAddressGroup endpoint1 = makeAddress("endpoint-addr-1"); + EquivalentAddressGroup endpoint2 = makeAddress("endpoint-addr-2"); + EquivalentAddressGroup endpoint3 = makeAddress("endpoint-addr-3"); + EquivalentAddressGroup endpoint4 = makeAddress("endpoint-addr-4"); + LocalityLbEndpoints localityLbEndpoints1 = + buildLocalityLbEndpoints(1, 70, ImmutableMap.of(endpoint1, true, endpoint2, true)); + LocalityLbEndpoints localityLbEndpoints2 = + buildLocalityLbEndpoints(1, 10, Collections.singletonMap(endpoint3, true)); + LocalityLbEndpoints localityLbEndpoints3 = + buildLocalityLbEndpoints(2, 20, Collections.singletonMap(endpoint4, true)); + xdsClient.deliverClusterLoadAssignment( + EDS_SERVICE_NAME, + ImmutableMap.of( + locality1, localityLbEndpoints1, + locality2, localityLbEndpoints2, + locality3, localityLbEndpoints3)); + assertThat(downstreamBalancers).hasSize(1); + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(downstreamBalancers); + assertThat(childBalancer.name).isEqualTo(PRIORITY_POLICY_NAME); + PriorityLbConfig config = (PriorityLbConfig) childBalancer.config; + assertThat(config.priorities).containsExactly("priority1", "priority2"); + PolicySelection child1 = config.childConfigs.get("priority1"); + assertThat(child1.getProvider().getPolicyName()).isEqualTo(WEIGHTED_TARGET_POLICY_NAME); + WeightedTargetConfig childConfig1 = (WeightedTargetConfig) child1.getConfig(); + assertThat(childConfig1.targets).hasSize(2); + WeightedPolicySelection target1 = childConfig1.targets.get(locality1.toString()); + assertThat(target1.weight).isEqualTo(70); + assertThat(target1.policySelection.getProvider().getPolicyName()).isEqualTo(LRS_POLICY_NAME); + assertLrsConfig((LrsConfig) target1.policySelection.getConfig(), CLUSTER, EDS_SERVICE_NAME, + LRS_SERVER_NAME, locality1, "round_robin"); + WeightedPolicySelection target2 = childConfig1.targets.get(locality2.toString()); + assertThat(target2.weight).isEqualTo(10); + assertThat(target2.policySelection.getProvider().getPolicyName()).isEqualTo(LRS_POLICY_NAME); + assertLrsConfig((LrsConfig) target2.policySelection.getConfig(), CLUSTER, EDS_SERVICE_NAME, + LRS_SERVER_NAME, locality2, "round_robin"); + + PolicySelection child2 = config.childConfigs.get("priority2"); + assertThat(child2.getProvider().getPolicyName()).isEqualTo(WEIGHTED_TARGET_POLICY_NAME); + WeightedTargetConfig childConfig2 = (WeightedTargetConfig) child2.getConfig(); + assertThat(childConfig2.targets).hasSize(1); + WeightedPolicySelection target3 = childConfig2.targets.get(locality3.toString()); + assertThat(target3.weight).isEqualTo(20); + assertThat(target3.policySelection.getProvider().getPolicyName()).isEqualTo(LRS_POLICY_NAME); + assertLrsConfig((LrsConfig) target3.policySelection.getConfig(), CLUSTER, EDS_SERVICE_NAME, + LRS_SERVER_NAME, locality3, "round_robin"); + + List priorityAddr1 = + AddressFilter.filter(childBalancer.addresses, "priority1"); + assertThat(priorityAddr1).hasSize(3); + assertAddressesEqual( + Arrays.asList(endpoint1, endpoint2, endpoint3), + priorityAddr1); + assertAddressesEqual( + Arrays.asList(endpoint1, endpoint2), + AddressFilter.filter(priorityAddr1, locality1.toString())); + assertAddressesEqual( + Collections.singletonList(endpoint3), + AddressFilter.filter(priorityAddr1, locality2.toString())); + + List priorityAddr2 = + AddressFilter.filter(childBalancer.addresses, "priority2"); + assertThat(priorityAddr2).hasSize(1); + assertAddressesEqual(Collections.singletonList(endpoint4), priorityAddr2); + assertAddressesEqual( + Collections.singletonList(endpoint4), + AddressFilter.filter(priorityAddr2, locality3.toString())); + } + + @Test + public void endpointResourceUpdated() { + EquivalentAddressGroup endpoint1 = makeAddress("endpoint-addr-1"); + LocalityLbEndpoints localityLbEndpoints1 = + buildLocalityLbEndpoints(1, 10, Collections.singletonMap(endpoint1, true)); + xdsClient.deliverClusterLoadAssignment( + EDS_SERVICE_NAME, ImmutableMap.of(locality1, localityLbEndpoints1)); + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(downstreamBalancers); + + assertThat(childBalancer.name).isEqualTo(PRIORITY_POLICY_NAME); + PriorityLbConfig config = (PriorityLbConfig) childBalancer.config; + assertThat(config.priorities).containsExactly("priority1"); + PolicySelection child = config.childConfigs.get("priority1"); + assertThat(child.getProvider().getPolicyName()).isEqualTo(WEIGHTED_TARGET_POLICY_NAME); + WeightedTargetConfig childConfig = (WeightedTargetConfig) child.getConfig(); + assertThat(childConfig.targets).hasSize(1); + WeightedPolicySelection target = childConfig.targets.get(locality1.toString()); + assertThat(target.weight).isEqualTo(10); + assertThat(target.policySelection.getProvider().getPolicyName()).isEqualTo(LRS_POLICY_NAME); + assertLrsConfig((LrsConfig) target.policySelection.getConfig(), CLUSTER, EDS_SERVICE_NAME, + LRS_SERVER_NAME, locality1, "round_robin"); + + List priorityAddr = + AddressFilter.filter(childBalancer.addresses, "priority1"); + assertThat(priorityAddr).hasSize(1); + assertAddressesEqual(Collections.singletonList(endpoint1), priorityAddr); + assertAddressesEqual( + Collections.singletonList(endpoint1), + AddressFilter.filter(priorityAddr, locality1.toString())); + + EquivalentAddressGroup endpoint2 = makeAddress("endpoint-addr-2"); + LocalityLbEndpoints localityLbEndpoints2 = + buildLocalityLbEndpoints(1, 30, Collections.singletonMap(endpoint2, true)); + xdsClient.deliverClusterLoadAssignment( + EDS_SERVICE_NAME, ImmutableMap.of(locality2, localityLbEndpoints2)); + + config = (PriorityLbConfig) childBalancer.config; + assertThat(config.priorities).containsExactly("priority1"); + child = config.childConfigs.get("priority1"); + assertThat(child.getProvider().getPolicyName()).isEqualTo(WEIGHTED_TARGET_POLICY_NAME); + childConfig = (WeightedTargetConfig) child.getConfig(); + assertThat(childConfig.targets).hasSize(1); + target = childConfig.targets.get(locality2.toString()); + assertThat(target.weight).isEqualTo(30); + assertThat(target.policySelection.getProvider().getPolicyName()).isEqualTo(LRS_POLICY_NAME); + assertLrsConfig((LrsConfig) target.policySelection.getConfig(), CLUSTER, EDS_SERVICE_NAME, + LRS_SERVER_NAME, locality2, "round_robin"); + + priorityAddr = AddressFilter.filter(childBalancer.addresses, "priority1"); + assertThat(priorityAddr).hasSize(1); + assertAddressesEqual(Collections.singletonList(endpoint2), priorityAddr); + assertAddressesEqual( + Collections.singletonList(endpoint2), + AddressFilter.filter(priorityAddr, locality2.toString())); + } + + @Test + public void endpointResourceNeverExist() { + xdsClient.deliverResourceNotFound(EDS_SERVICE_NAME); + assertThat(currentState).isEqualTo(ConnectivityState.TRANSIENT_FAILURE); + PickResult result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); + assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAVAILABLE); + assertThat(result.getStatus().getDescription()) + .isEqualTo("Resource " + EDS_SERVICE_NAME + " is unavailable"); + } + + @Test + public void endpointResourceRemoved() { + deliverSimpleClusterLoadAssignment(EDS_SERVICE_NAME); + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(downstreamBalancers); + assertThat(childBalancer.shutdown).isFalse(); + + xdsClient.deliverResourceNotFound(EDS_SERVICE_NAME); + assertThat(childBalancer.shutdown).isTrue(); + assertThat(currentState).isEqualTo(ConnectivityState.TRANSIENT_FAILURE); + PickResult result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); + assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAVAILABLE); + assertThat(result.getStatus().getDescription()) + .isEqualTo("Resource " + EDS_SERVICE_NAME + " is unavailable"); + } + + @Test + public void handleEndpointResource_ignoreUnhealthyEndpoints() { + EquivalentAddressGroup endpoint1 = makeAddress("endpoint-addr-1"); + EquivalentAddressGroup endpoint2 = makeAddress("endpoint-addr-2"); + LocalityLbEndpoints localityLbEndpoints = + buildLocalityLbEndpoints(1, 10, ImmutableMap.of(endpoint1, false, endpoint2, true)); + xdsClient.deliverClusterLoadAssignment( + EDS_SERVICE_NAME, Collections.singletonMap(locality1, localityLbEndpoints)); + + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(downstreamBalancers); + PriorityLbConfig config = (PriorityLbConfig) childBalancer.config; + PolicySelection child = config.childConfigs.get("priority1"); + WeightedTargetConfig childConfig = (WeightedTargetConfig) child.getConfig(); + assertThat(childConfig.targets.keySet()).containsExactly(locality1.toString()); + + List priorityAddr = + AddressFilter.filter(childBalancer.addresses, "priority1"); + assertThat(priorityAddr).hasSize(1); + assertAddressesEqual(Collections.singletonList(endpoint2), priorityAddr); + assertAddressesEqual( + Collections.singletonList(endpoint2), + AddressFilter.filter(priorityAddr, locality1.toString())); + } + + @Test + public void handleEndpointResource_ignoreLocalitiesWithNoHealthyEndpoints() { + EquivalentAddressGroup endpoint1 = makeAddress("endpoint-addr-1"); + EquivalentAddressGroup endpoint2 = makeAddress("endpoint-addr-2"); + LocalityLbEndpoints localityLbEndpoints1 = + buildLocalityLbEndpoints(1, 10, Collections.singletonMap(endpoint1, false)); + LocalityLbEndpoints localityLbEndpoints2 = + buildLocalityLbEndpoints(1, 10, Collections.singletonMap(endpoint2, true)); + xdsClient.deliverClusterLoadAssignment( + EDS_SERVICE_NAME, + ImmutableMap.of(locality1, localityLbEndpoints1, locality2, localityLbEndpoints2)); + + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(downstreamBalancers); + PriorityLbConfig config = (PriorityLbConfig) childBalancer.config; + PolicySelection child = config.childConfigs.get("priority1"); + WeightedTargetConfig childConfig = (WeightedTargetConfig) child.getConfig(); + assertThat(childConfig.targets.keySet()).containsExactly(locality2.toString()); + + List priorityAddr = + AddressFilter.filter(childBalancer.addresses, "priority1"); + assertThat(priorityAddr).hasSize(1); + assertAddressesEqual(Collections.singletonList(endpoint2), priorityAddr); + assertAddressesEqual( + Collections.singletonList(endpoint2), + AddressFilter.filter(priorityAddr, locality2.toString())); + } + + @Test + public void handleEndpointResource_ignorePrioritiesWithNoHealthyEndpoints() { + EquivalentAddressGroup endpoint1 = makeAddress("endpoint-addr-1"); + EquivalentAddressGroup endpoint2 = makeAddress("endpoint-addr-2"); + LocalityLbEndpoints localityLbEndpoints1 = + buildLocalityLbEndpoints(1, 10, Collections.singletonMap(endpoint1, false)); + LocalityLbEndpoints localityLbEndpoints2 = + buildLocalityLbEndpoints(2, 10, Collections.singletonMap(endpoint2, true)); + xdsClient.deliverClusterLoadAssignment( + EDS_SERVICE_NAME, + ImmutableMap.of(locality1, localityLbEndpoints1, locality2, localityLbEndpoints2)); + + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(downstreamBalancers); + PriorityLbConfig config = (PriorityLbConfig) childBalancer.config; + assertThat(config.priorities).containsExactly("priority2"); + + List priorityAddr = + AddressFilter.filter(childBalancer.addresses, "priority2"); + assertThat(priorityAddr).hasSize(1); + assertAddressesEqual(Collections.singletonList(endpoint2), priorityAddr); + assertAddressesEqual( + Collections.singletonList(endpoint2), + AddressFilter.filter(priorityAddr, locality2.toString())); + } + + @Test + public void handleEndpointResource_errorIfNoUsableEndpoints() { + EquivalentAddressGroup endpoint1 = makeAddress("endpoint-addr-1"); + EquivalentAddressGroup endpoint2 = makeAddress("endpoint-addr-2"); + LocalityLbEndpoints localityLbEndpoints1 = + buildLocalityLbEndpoints(1, 10, Collections.singletonMap(endpoint1, false)); + LocalityLbEndpoints localityLbEndpoints2 = + buildLocalityLbEndpoints(2, 10, Collections.singletonMap(endpoint2, false)); + xdsClient.deliverClusterLoadAssignment( + EDS_SERVICE_NAME, + ImmutableMap.of(locality1, localityLbEndpoints1, locality2, localityLbEndpoints2)); + + assertThat(downstreamBalancers).isEmpty(); + assertThat(currentState).isEqualTo(ConnectivityState.TRANSIENT_FAILURE); + PickResult result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); + assertThat(result.getStatus().isOk()).isFalse(); + assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAVAILABLE); + assertThat(result.getStatus().getDescription()) + .isEqualTo("No usable priority/locality/endpoint"); + } + + @Test + public void handleDrops() { + FakeLoadBalancerProvider fakeRoundRobinProvider = new FakeLoadBalancerProvider("round_robin"); + prepareRealDownstreamLbPolicies(fakeRoundRobinProvider); + when(mockRandom.nextInt(anyInt())).thenReturn(499_999, 1_000_000); + EquivalentAddressGroup endpoint1 = makeAddress("endpoint-addr-1"); + LocalityLbEndpoints localityLbEndpoints1 = + buildLocalityLbEndpoints(1, 10, Collections.singletonMap(endpoint1, true)); + xdsClient.deliverClusterLoadAssignment( + EDS_SERVICE_NAME, + Collections.singletonList(new DropOverload("throttle", 500_000)), + Collections.singletonMap(locality1, localityLbEndpoints1)); + assertThat(downstreamBalancers).hasSize(1); // one leaf balancer + FakeLoadBalancer leafBalancer = Iterables.getOnlyElement(downstreamBalancers); + assertThat(leafBalancer.name).isEqualTo("round_robin"); + assertAddressesEqual(Collections.singletonList(makeAddress("endpoint-addr-1")), + leafBalancer.addresses); + Subchannel subchannel = leafBalancer.helper.createSubchannel( + CreateSubchannelArgs.newBuilder().setAddresses(leafBalancer.addresses).build()); + leafBalancer.deliverSubchannelState(subchannel, ConnectivityState.READY); + assertThat(currentState).isEqualTo(ConnectivityState.READY); + PickResult result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); + assertThat(result.getStatus().isOk()).isFalse(); + assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAVAILABLE); + assertThat(result.getStatus().getDescription()).isEqualTo("Dropped: throttle"); + assertThat(xdsClient.dropStats.get(EDS_SERVICE_NAME).get("throttle").get()).isEqualTo(1); + + result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); + assertThat(result.getStatus().isOk()).isTrue(); + assertThat(result.getSubchannel()).isSameInstanceAs(subchannel); + } + + @Test + public void configUpdate_changeEdsServiceName_afterChildPolicyReady_switchGracefully() { + deliverSimpleClusterLoadAssignment(EDS_SERVICE_NAME); // downstream LB polices instantiated + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(downstreamBalancers); + Subchannel subchannel1 = mock(Subchannel.class); + childBalancer.deliverSubchannelState(subchannel1, ConnectivityState.READY); + assertThat(currentState).isEqualTo(ConnectivityState.READY); + PickResult result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); + assertThat(result.getSubchannel()).isSameInstanceAs(subchannel1); + + String newEdsServiceName = "service-foo.googleapis.com"; + loadBalancer.handleResolvedAddresses( + ResolvedAddresses.newBuilder() + .setAddresses(Collections.emptyList()) + .setAttributes( + Attributes.newBuilder().set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool).build()) + .setLoadBalancingPolicyConfig( + new EdsConfig( + CLUSTER, newEdsServiceName, LRS_SERVER_NAME, weightedTarget, roundRobin)) + .build()); + deliverSimpleClusterLoadAssignment(newEdsServiceName); // instantiate the new subtree + assertThat(downstreamBalancers).hasSize(2); + FakeLoadBalancer newChildBalancer = downstreamBalancers.get(1); + assertThat(currentState).isEqualTo(ConnectivityState.READY); + result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); + assertThat(result.getSubchannel()).isSameInstanceAs(subchannel1); + Subchannel subchannel2 = mock(Subchannel.class); + newChildBalancer.deliverSubchannelState(subchannel2, ConnectivityState.READY); + assertThat(childBalancer.shutdown).isTrue(); + result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); + assertThat(result.getSubchannel()).isSameInstanceAs(subchannel2); + } + + @Test + public void configUpdate_changeEndpointPickingPolicy() { + FakeLoadBalancerProvider fakeRoundRobinProvider = new FakeLoadBalancerProvider("round_robin"); + prepareRealDownstreamLbPolicies(fakeRoundRobinProvider); + deliverSimpleClusterLoadAssignment(EDS_SERVICE_NAME); // downstream LB policies instantiated + FakeLoadBalancer leafBalancer = Iterables.getOnlyElement(downstreamBalancers); + assertThat(leafBalancer.name).isEqualTo("round_robin"); + FakeLoadBalancerProvider fakePickFirstProvider = new FakeLoadBalancerProvider("pick_first"); + PolicySelection fakePickFirstSelection = + new PolicySelection(fakePickFirstProvider, null, null); + loadBalancer.handleResolvedAddresses(ResolvedAddresses.newBuilder() + .setAddresses(Collections.emptyList()) + .setAttributes( + Attributes.newBuilder().set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool).build()) + .setLoadBalancingPolicyConfig( + new EdsConfig( + CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_NAME, weightedTarget, fakePickFirstSelection)) + .build()); + assertThat(leafBalancer.shutdown).isTrue(); + leafBalancer = Iterables.getOnlyElement(downstreamBalancers); + assertThat(leafBalancer.name).isEqualTo("pick_first"); + } + + @Test + public void endpointDiscoveryError_beforeChildPolicyInstantiated_propagateToUpstream() { + xdsClient.deliverError(Status.UNAUTHENTICATED.withDescription("permission denied")); + assertThat(currentState).isEqualTo(ConnectivityState.TRANSIENT_FAILURE); + PickResult result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); + assertThat(result.getStatus().isOk()).isFalse(); + assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAUTHENTICATED); + assertThat(result.getStatus().getDescription()).isEqualTo("permission denied"); + } + + @Test + public void endpointDiscoveryError_afterChildPolicyInstantiated_keepUsingCurrentEndpoints() { + deliverSimpleClusterLoadAssignment(EDS_SERVICE_NAME); + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(downstreamBalancers); + xdsClient.deliverError(Status.UNAVAILABLE.withDescription("not found")); + + assertThat(currentState).isEqualTo(ConnectivityState.CONNECTING); + PickResult result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); + assertThat(result.getStatus().isOk()).isTrue(); + assertThat(childBalancer.shutdown).isFalse(); + } + + @Test + public void nameResolutionError_beforeChildPolicyInstantiated_returnErrorPickerToUpstream() { + loadBalancer.handleNameResolutionError(Status.UNIMPLEMENTED.withDescription("not found")); + assertThat(currentState).isEqualTo(ConnectivityState.TRANSIENT_FAILURE); + PickResult result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); + assertThat(result.getStatus().isOk()).isFalse(); + assertThat(result.getStatus().getCode()).isEqualTo(Code.UNIMPLEMENTED); + assertThat(result.getStatus().getDescription()).isEqualTo("not found"); + } + + @Test + public void nameResolutionError_afterChildPolicyInstantiated_propagateToDownstream() { + deliverSimpleClusterLoadAssignment(EDS_SERVICE_NAME); + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(downstreamBalancers); + + loadBalancer.handleNameResolutionError( + Status.UNAVAILABLE.withDescription("cannot reach server")); + assertThat(childBalancer.upstreamError.getCode()).isEqualTo(Code.UNAVAILABLE); + assertThat(childBalancer.upstreamError.getDescription()) + .isEqualTo("cannot reach server"); + } + + @Test + public void generatePriorityLbConfig() { + Map> prioritizedLocalityWeights = new HashMap<>(); + prioritizedLocalityWeights.put(1, ImmutableMap.of(locality1, 20, locality2, 50)); + prioritizedLocalityWeights.put(2, ImmutableMap.of(locality3, 30)); + PriorityLbConfig config = + EdsLoadBalancer2.generatePriorityLbConfig( + CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_NAME, weightedTarget, roundRobin, registry, + prioritizedLocalityWeights); + assertThat(config.childConfigs).hasSize(2); + assertThat(config.priorities).containsExactly("priority1", "priority2"); + PolicySelection child1 = config.childConfigs.get("priority1"); + assertThat(child1.getProvider().getPolicyName()).isEqualTo(WEIGHTED_TARGET_POLICY_NAME); + WeightedTargetConfig weightedTargetConfig1 = (WeightedTargetConfig) child1.getConfig(); + assertThat(weightedTargetConfig1.targets).hasSize(2); + WeightedPolicySelection childTarget1 = weightedTargetConfig1.targets.get(locality1.toString()); + assertThat(childTarget1.weight).isEqualTo(20); + assertThat(childTarget1.policySelection.getProvider().getPolicyName()) + .isEqualTo(LRS_POLICY_NAME); + assertLrsConfig((LrsConfig) childTarget1.policySelection.getConfig(), CLUSTER, + EDS_SERVICE_NAME, LRS_SERVER_NAME, locality1, "round_robin"); + WeightedPolicySelection childTarget2 = weightedTargetConfig1.targets.get(locality2.toString()); + assertThat(childTarget2.weight).isEqualTo(50); + assertThat(childTarget2.policySelection.getProvider().getPolicyName()) + .isEqualTo(LRS_POLICY_NAME); + assertLrsConfig((LrsConfig) childTarget2.policySelection.getConfig(), CLUSTER, + EDS_SERVICE_NAME, LRS_SERVER_NAME, locality2, "round_robin"); + + PolicySelection child2 = config.childConfigs.get("priority2"); + assertThat(child2.getProvider().getPolicyName()).isEqualTo(WEIGHTED_TARGET_POLICY_NAME); + WeightedTargetConfig weightedTargetConfig2 = (WeightedTargetConfig) child2.getConfig(); + assertThat(weightedTargetConfig2.targets).hasSize(1); + WeightedPolicySelection childTarget3 = weightedTargetConfig2.targets.get(locality3.toString()); + assertThat(childTarget3.weight).isEqualTo(30); + assertThat(childTarget3.policySelection.getProvider().getPolicyName()) + .isEqualTo(LRS_POLICY_NAME); + assertLrsConfig((LrsConfig) childTarget3.policySelection.getConfig(), CLUSTER, + EDS_SERVICE_NAME, LRS_SERVER_NAME, locality3, "round_robin"); + } + + @Test + public void generateWeightedTargetLbConfig_withLrsPolicy() { + Map localityWeights = ImmutableMap.of(locality1, 30, locality2, 40); + WeightedTargetConfig config = + EdsLoadBalancer2.generateWeightedTargetLbConfig( + CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_NAME, roundRobin, registry, localityWeights); + assertThat(config.targets).hasSize(2); + WeightedPolicySelection target1 = config.targets.get(locality1.toString()); + assertThat(target1.weight).isEqualTo(30); + assertThat(target1.policySelection.getProvider().getPolicyName()).isEqualTo(LRS_POLICY_NAME); + assertLrsConfig((LrsConfig) target1.policySelection.getConfig(), CLUSTER, EDS_SERVICE_NAME, + LRS_SERVER_NAME, locality1, "round_robin"); + + WeightedPolicySelection target2 = config.targets.get(locality2.toString()); + assertThat(target2.weight).isEqualTo(40); + assertThat(target2.policySelection.getProvider().getPolicyName()).isEqualTo(LRS_POLICY_NAME); + assertLrsConfig((LrsConfig) target2.policySelection.getConfig(), CLUSTER, EDS_SERVICE_NAME, + LRS_SERVER_NAME, locality2, "round_robin"); + } + + @Test + public void generateWeightedTargetLbConfig_withoutLrsPolicy() { + Map localityWeights = ImmutableMap.of(locality1, 30, locality2, 40); + WeightedTargetConfig config = + EdsLoadBalancer2.generateWeightedTargetLbConfig( + CLUSTER, EDS_SERVICE_NAME, null, roundRobin, registry, localityWeights); + assertThat(config.targets).hasSize(2); + WeightedPolicySelection target1 = config.targets.get(locality1.toString()); + assertThat(target1.weight).isEqualTo(30); + assertThat(target1.policySelection.getProvider().getPolicyName()).isEqualTo("round_robin"); + + WeightedPolicySelection target2 = config.targets.get(locality2.toString()); + assertThat(target2.weight).isEqualTo(40); + assertThat(target2.policySelection.getProvider().getPolicyName()).isEqualTo("round_robin"); + } + + private void deliverSimpleClusterLoadAssignment(String resourceName) { + EquivalentAddressGroup endpoint1 = makeAddress("endpoint-addr-1"); + LocalityLbEndpoints localityLbEndpoints1 = + buildLocalityLbEndpoints(1, 10, Collections.singletonMap(endpoint1, true)); + xdsClient.deliverClusterLoadAssignment( + resourceName, + Collections.singletonMap(locality1, localityLbEndpoints1)); + } + + /** + * Instantiates the downstream LB policy subtree with real implementations, except the leaf + * policy is replaced with a fake implementation to avoid creating connections. + */ + private void prepareRealDownstreamLbPolicies(FakeLoadBalancerProvider fakeLeafPolicyProvider) { + registry.deregister(registry.getProvider(PRIORITY_POLICY_NAME)); + registry.register(new PriorityLoadBalancerProvider()); + registry.deregister(registry.getProvider(LRS_POLICY_NAME)); + registry.register(new LrsLoadBalancerProvider()); + PolicySelection weightedTargetSelection = + new PolicySelection(new WeightedTargetLoadBalancerProvider(), null, null); + PolicySelection fakeLeafPolicySelection = + new PolicySelection(fakeLeafPolicyProvider, null, null); + loadBalancer.handleResolvedAddresses( + ResolvedAddresses.newBuilder() + .setAddresses(Collections.emptyList()) + .setAttributes( + Attributes.newBuilder().set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool).build()) + .setLoadBalancingPolicyConfig( + new EdsConfig(CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_NAME, weightedTargetSelection, + fakeLeafPolicySelection)) + .build()); + } + + private static void assertLrsConfig( + LrsConfig config, String cluster, String edsServiceName, String lrsServerName, + Locality locality, String childPolicy) { + assertThat(config.clusterName).isEqualTo(cluster); + assertThat(config.edsServiceName).isEqualTo(edsServiceName); + assertThat(config.lrsServerName).isEqualTo(lrsServerName); + assertThat(config.locality).isEqualTo(locality); + assertThat(config.childPolicy.getProvider().getPolicyName()).isEqualTo(childPolicy); + } + + /** Asserts two list of EAGs contains same addresses, regardless of attributes. */ + private static void assertAddressesEqual( + List expected, List actual) { + assertThat(actual.size()).isEqualTo(expected.size()); + for (int i = 0; i < actual.size(); i++) { + assertThat(actual.get(i).getAddresses()).isEqualTo(expected.get(i).getAddresses()); + } + } + + private static LocalityLbEndpoints buildLocalityLbEndpoints( + int priority, int localityWeight, Map managedEndpoints) { + List endpoints = new ArrayList<>(); + for (EquivalentAddressGroup addr : managedEndpoints.keySet()) { + boolean status = managedEndpoints.get(addr); + endpoints.add(new LbEndpoint(addr, 100 /* used */, status)); + } + return new LocalityLbEndpoints(endpoints, localityWeight, priority); + } + + private static EquivalentAddressGroup makeAddress(final String name) { + class FakeSocketAddress extends SocketAddress { + private final String name; + + private FakeSocketAddress(String name) { + this.name = name; + } + + @Override + public int hashCode() { + return Objects.hash(name); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof FakeSocketAddress)) { + return false; + } + FakeSocketAddress that = (FakeSocketAddress) o; + return Objects.equals(name, that.name); + } + + @Override + public String toString() { + return name; + } + } + + return new EquivalentAddressGroup(new FakeSocketAddress(name)); + } + + private final class FakeXdsClient extends XdsClient { + private final Map watchers = new HashMap<>(); + private final Map> dropStats = new HashMap<>(); + + @Override + void shutdown() { + // no-op + } + + @Override + void watchEndpointData(String clusterName, EndpointWatcher watcher) { + watchers.put(clusterName, watcher); + } + + @Override + void cancelEndpointDataWatch(String clusterName, EndpointWatcher watcher) { + watchers.remove(clusterName); + } + + @Override + LoadStatsStore addClientStats(String clusterName, @Nullable String clusterServiceName) { + ConcurrentMap dropCounters = new ConcurrentHashMap<>(); + dropStats.put(clusterServiceName, dropCounters); + return new LoadStatsStoreImpl(clusterName, clusterServiceName, + fakeClock.getStopwatchSupplier().get(), dropCounters); + } + + @Override + void removeClientStats(String clusterName, @Nullable String clusterServiceName) { + dropStats.remove(clusterServiceName); + } + + void deliverClusterLoadAssignment( + String resource, Map localityLbEndpointsMap) { + deliverClusterLoadAssignment( + resource, Collections.emptyList(), localityLbEndpointsMap); + } + + void deliverClusterLoadAssignment( + final String resource, final List dropOverloads, + final Map localityLbEndpointsMap) { + syncContext.execute(new Runnable() { + @Override + public void run() { + if (watchers.containsKey(resource)) { + EndpointUpdate.Builder builder = EndpointUpdate.newBuilder().setClusterName(resource); + for (DropOverload dropOverload : dropOverloads) { + builder.addDropPolicy(dropOverload); + } + for (Locality locality : localityLbEndpointsMap.keySet()) { + builder.addLocalityLbEndpoints(locality, localityLbEndpointsMap.get(locality)); + } + watchers.get(resource).onEndpointChanged(builder.build()); + } + } + }); + } + + void deliverResourceNotFound(final String resource) { + syncContext.execute(new Runnable() { + @Override + public void run() { + if (watchers.containsKey(resource)) { + watchers.get(resource).onResourceDoesNotExist(resource); + } + } + }); + } + + void deliverError(final Status error) { + syncContext.execute(new Runnable() { + @Override + public void run() { + for (EndpointWatcher watcher : watchers.values()) { + watcher.onError(error); + } + } + }); + } + } + + private final class FakeLoadBalancerProvider extends LoadBalancerProvider { + private final String policyName; + + FakeLoadBalancerProvider(String policyName) { + this.policyName = policyName; + } + + @Override + public LoadBalancer newLoadBalancer(Helper helper) { + FakeLoadBalancer balancer = new FakeLoadBalancer(policyName, helper); + downstreamBalancers.add(balancer); + return balancer; + } + + @Override + public boolean isAvailable() { + return true; + } + + @Override + public int getPriority() { + return 0; // doesn't matter + } + + @Override + public String getPolicyName() { + return policyName; + } + } + + private final class FakeLoadBalancer extends LoadBalancer { + private final String name; + private final Helper helper; + private List addresses; + private Object config; + private Status upstreamError; + private boolean shutdown; + + FakeLoadBalancer(String name, Helper helper) { + this.name = name; + this.helper = helper; + } + + @Override + public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { + addresses = resolvedAddresses.getAddresses(); + config = resolvedAddresses.getLoadBalancingPolicyConfig(); + } + + @Override + public void handleNameResolutionError(Status error) { + upstreamError = error; + } + + @Override + public void shutdown() { + shutdown = true; + downstreamBalancers.remove(this); + } + + void deliverSubchannelState(final Subchannel subchannel, ConnectivityState state) { + SubchannelPicker picker = new SubchannelPicker() { + @Override + public PickResult pickSubchannel(PickSubchannelArgs args) { + return PickResult.withSubchannel(subchannel); + } + }; + helper.updateBalancingState(state, picker); + } + } + + private final class FakeLbHelper extends LoadBalancer.Helper { + + @Override + public void updateBalancingState( + @Nonnull ConnectivityState newState, @Nonnull SubchannelPicker newPicker) { + currentState = newState; + currentPicker = newPicker; + } + + @Override + public Subchannel createSubchannel(CreateSubchannelArgs args) { + return mock(Subchannel.class); + } + + @Override + public SynchronizationContext getSynchronizationContext() { + return syncContext; + } + + @Override + public ScheduledExecutorService getScheduledExecutorService() { + return fakeClock.getScheduledExecutorService(); + } + + @Override + public ManagedChannel createOobChannel(EquivalentAddressGroup eag, String authority) { + throw new UnsupportedOperationException("should not be called"); + } + + @Deprecated + @Override + public NameResolver.Factory getNameResolverFactory() { + throw new UnsupportedOperationException("should not be called"); + } + + @Override + public String getAuthority() { + return AUTHORITY; + } + } +} diff --git a/xds/src/test/java/io/grpc/xds/EdsLoadBalancerProviderTest.java b/xds/src/test/java/io/grpc/xds/EdsLoadBalancerProviderTest.java index 6c69afcc78f..24d25939966 100644 --- a/xds/src/test/java/io/grpc/xds/EdsLoadBalancerProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/EdsLoadBalancerProviderTest.java @@ -18,11 +18,7 @@ import static com.google.common.truth.Truth.assertThat; -import io.grpc.NameResolver.ConfigOrError; -import io.grpc.internal.JsonParser; -import io.grpc.xds.EdsLoadBalancerProvider.EdsConfig; -import java.io.IOException; -import java.util.Map; +import io.grpc.LoadBalancerRegistry; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -33,44 +29,13 @@ public class EdsLoadBalancerProviderTest { private final EdsLoadBalancerProvider provider = new EdsLoadBalancerProvider(); @Test - public void parseEdsLoadBalancingPolicyConfig() throws IOException { - String rawEdsLbConfig = "{\n" - + " \"cluster\": \"cluster-foo.googleapis.com\",\n" - + " \"edsServiceName\": \"cluster-foo:service-blade\",\n" - + " \"lrsLoadReportingServerName\": \"trafficdirector.googleapis.com\",\n" - + " \"endpointPickingPolicy\": [\n" - + " { \"policy_foo\": {} },\n" - + " { \"pick_first\": {} }\n" - + " ]\n" - + "}"; - - @SuppressWarnings("unchecked") - Map rawLbConfigMap = (Map) JsonParser.parse(rawEdsLbConfig); - ConfigOrError result = provider.parseLoadBalancingPolicyConfig(rawLbConfigMap); - assertThat(result.getConfig()).isNotNull(); - EdsConfig config = (EdsConfig) result.getConfig(); - assertThat(config.clusterName).isEqualTo("cluster-foo.googleapis.com"); - assertThat(config.edsServiceName).isEqualTo("cluster-foo:service-blade"); - assertThat(config.lrsServerName).isEqualTo("trafficdirector.googleapis.com"); - assertThat(config.endpointPickingPolicy.getProvider().getPolicyName()) - .isEqualTo("pick_first"); + public void isAvailable() { + assertThat(provider.isAvailable()).isTrue(); } @Test - public void parseEdsLoadBalancingPolicyConfig_defaultEndpointPickingPolicy_roundRobin() - throws IOException { - String rawEdsLbConfig = "{\n" - + " \"cluster\": \"cluster-foo.googleapis.com\",\n" - + " \"edsServiceName\": \"cluster-foo:service-blade\",\n" - + " \"lrsLoadReportingServerName\": \"trafficdirector.googleapis.com\"\n" - + "}"; - - @SuppressWarnings("unchecked") - Map rawLbConfigMap = (Map) JsonParser.parse(rawEdsLbConfig); - ConfigOrError result = provider.parseLoadBalancingPolicyConfig(rawLbConfigMap); - assertThat(result.getConfig()).isNotNull(); - EdsConfig config = (EdsConfig) result.getConfig(); - assertThat(config.endpointPickingPolicy.getProvider().getPolicyName()) - .isEqualTo("round_robin"); + public void provided() { + LoadBalancerRegistry lbRegistry = LoadBalancerRegistry.getDefaultRegistry(); + assertThat(lbRegistry.getProvider(XdsLbPolicies.EDS_POLICY_NAME)).isNotNull(); } } From b31d6830a2af3196e588217320d2781112b7cf73 Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Fri, 18 Sep 2020 17:55:29 -0700 Subject: [PATCH 54/86] xds: fail to create xDS channel if no server with supported channel creds found (#7400) Create the xDS channel outside the XdsClient. Throw an XdsInitializationException if the provided server list (parsed from the bootstrap file) can not be used to create such a channel. The exception is caught by the xDS resolver and propagated to the Channel gracefully as a name resolution error. --- .../java/io/grpc/xds/EdsLoadBalancer.java | 17 +-- .../java/io/grpc/xds/XdsChannelFactory.java | 100 ++++++++++++++++++ xds/src/main/java/io/grpc/xds/XdsClient.java | 68 ------------ .../main/java/io/grpc/xds/XdsClientImpl.java | 9 +- .../xds/XdsClientWrapperForServerSds.java | 9 +- .../java/io/grpc/xds/XdsNameResolver.java | 15 ++- .../io/grpc/xds/XdsNameResolverProvider.java | 24 ++--- .../java/io/grpc/xds/EdsLoadBalancerTest.java | 4 +- .../io/grpc/xds/XdsChannelFactoryTest.java | 99 +++++++++++++++++ .../java/io/grpc/xds/XdsClientImplTest.java | 18 +--- .../xds/XdsClientImplTestForListener.java | 19 +--- .../java/io/grpc/xds/XdsClientImplTestV2.java | 18 +--- .../test/java/io/grpc/xds/XdsClientTest.java | 59 ----------- .../java/io/grpc/xds/XdsNameResolverTest.java | 52 ++++++++- 14 files changed, 284 insertions(+), 227 deletions(-) create mode 100644 xds/src/main/java/io/grpc/xds/XdsChannelFactory.java create mode 100644 xds/src/test/java/io/grpc/xds/XdsChannelFactoryTest.java diff --git a/xds/src/main/java/io/grpc/xds/EdsLoadBalancer.java b/xds/src/main/java/io/grpc/xds/EdsLoadBalancer.java index b009ea7d530..60652af299f 100644 --- a/xds/src/main/java/io/grpc/xds/EdsLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/EdsLoadBalancer.java @@ -32,7 +32,6 @@ import io.grpc.internal.ObjectPool; import io.grpc.util.GracefulSwitchLoadBalancer; import io.grpc.xds.Bootstrapper.BootstrapInfo; -import io.grpc.xds.Bootstrapper.ServerInfo; import io.grpc.xds.EdsLoadBalancerProvider.EdsConfig; import io.grpc.xds.EnvoyProtoData.DropOverload; import io.grpc.xds.EnvoyProtoData.Locality; @@ -43,7 +42,7 @@ import io.grpc.xds.XdsClient.EndpointUpdate; import io.grpc.xds.XdsClient.EndpointWatcher; import io.grpc.xds.XdsClient.RefCountedXdsClientObjectPool; -import io.grpc.xds.XdsClient.XdsChannelFactory; +import io.grpc.xds.XdsClient.XdsChannel; import io.grpc.xds.XdsClient.XdsClientFactory; import io.grpc.xds.XdsLogger.XdsLogLevel; import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; @@ -129,8 +128,10 @@ public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { xdsClientPool = attributes.get(XdsAttributes.XDS_CLIENT_POOL); if (xdsClientPool == null) { final BootstrapInfo bootstrapInfo; + final XdsChannel channel; try { bootstrapInfo = bootstrapper.readBootstrap(); + channel = channelFactory.createChannel(bootstrapInfo.getServers()); } catch (Exception e) { helper.updateBalancingState( TRANSIENT_FAILURE, @@ -139,24 +140,14 @@ public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { return; } - final List serverList = bootstrapInfo.getServers(); final Node node = bootstrapInfo.getNode(); - if (serverList.isEmpty()) { - helper.updateBalancingState( - TRANSIENT_FAILURE, - new ErrorPicker( - Status.UNAVAILABLE - .withDescription("No management server provided by bootstrap"))); - return; - } XdsClientFactory xdsClientFactory = new XdsClientFactory() { @Override XdsClient createXdsClient() { return new XdsClientImpl( helper.getAuthority(), - serverList, - channelFactory, + channel, node, helper.getSynchronizationContext(), helper.getScheduledExecutorService(), diff --git a/xds/src/main/java/io/grpc/xds/XdsChannelFactory.java b/xds/src/main/java/io/grpc/xds/XdsChannelFactory.java new file mode 100644 index 00000000000..2ba7a6f00b5 --- /dev/null +++ b/xds/src/main/java/io/grpc/xds/XdsChannelFactory.java @@ -0,0 +1,100 @@ +/* + * Copyright 2020 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.xds; + +import com.google.common.annotations.VisibleForTesting; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.alts.GoogleDefaultChannelBuilder; +import io.grpc.xds.Bootstrapper.ChannelCreds; +import io.grpc.xds.Bootstrapper.ServerInfo; +import io.grpc.xds.XdsClient.XdsChannel; +import io.grpc.xds.XdsLogger.XdsLogLevel; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Factory for creating channels to xDS severs. + */ +abstract class XdsChannelFactory { + @VisibleForTesting + static boolean experimentalV3SupportEnvVar = Boolean.parseBoolean( + System.getenv("GRPC_XDS_EXPERIMENTAL_V3_SUPPORT")); + + private static final String XDS_V3_SERVER_FEATURE = "xds_v3"; + private static final XdsChannelFactory DEFAULT_INSTANCE = new XdsChannelFactory() { + /** + * Creates a channel to the first server in the given list. + */ + @Override + XdsChannel createChannel(List servers) throws XdsInitializationException { + if (servers.isEmpty()) { + throw new XdsInitializationException("No server provided"); + } + XdsLogger logger = XdsLogger.withPrefix("xds-client-channel-factory"); + ServerInfo serverInfo = servers.get(0); + String serverUri = serverInfo.getServerUri(); + logger.log(XdsLogLevel.INFO, "Creating channel to {0}", serverUri); + List channelCredsList = serverInfo.getChannelCredentials(); + ManagedChannelBuilder channelBuilder = null; + // Use the first supported channel credentials configuration. + for (ChannelCreds creds : channelCredsList) { + switch (creds.getType()) { + case "google_default": + logger.log(XdsLogLevel.INFO, "Using channel credentials: google_default"); + channelBuilder = GoogleDefaultChannelBuilder.forTarget(serverUri); + break; + case "insecure": + logger.log(XdsLogLevel.INFO, "Using channel credentials: insecure"); + channelBuilder = ManagedChannelBuilder.forTarget(serverUri).usePlaintext(); + break; + case "tls": + logger.log(XdsLogLevel.INFO, "Using channel credentials: tls"); + channelBuilder = ManagedChannelBuilder.forTarget(serverUri); + break; + default: + } + if (channelBuilder != null) { + break; + } + } + if (channelBuilder == null) { + throw new XdsInitializationException("No server with supported channel creds found"); + } + + ManagedChannel channel = channelBuilder + .keepAliveTime(5, TimeUnit.MINUTES) + .build(); + boolean useProtocolV3 = experimentalV3SupportEnvVar + && serverInfo.getServerFeatures().contains(XDS_V3_SERVER_FEATURE); + + return new XdsChannel(channel, useProtocolV3); + } + }; + + static XdsChannelFactory getInstance() { + return DEFAULT_INSTANCE; + } + + /** + * Creates a channel to one of the provided management servers. + * + * @throws XdsInitializationException if failed to create a channel with the given list of + * servers. + */ + abstract XdsChannel createChannel(List servers) throws XdsInitializationException; +} diff --git a/xds/src/main/java/io/grpc/xds/XdsClient.java b/xds/src/main/java/io/grpc/xds/XdsClient.java index 7efb2a2937e..3e639c7fa3d 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClient.java +++ b/xds/src/main/java/io/grpc/xds/XdsClient.java @@ -16,7 +16,6 @@ package io.grpc.xds; -import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import com.google.common.annotations.VisibleForTesting; @@ -25,13 +24,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; import io.grpc.Status; -import io.grpc.alts.GoogleDefaultChannelBuilder; import io.grpc.internal.ObjectPool; -import io.grpc.xds.Bootstrapper.BootstrapInfo; -import io.grpc.xds.Bootstrapper.ChannelCreds; -import io.grpc.xds.Bootstrapper.ServerInfo; import io.grpc.xds.EnvoyProtoData.DropOverload; import io.grpc.xds.EnvoyProtoData.Locality; import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints; @@ -39,7 +33,6 @@ import io.grpc.xds.EnvoyServerProtoData.Listener; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import io.grpc.xds.LoadStatsManager.LoadStatsStore; -import io.grpc.xds.XdsLogger.XdsLogLevel; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -47,7 +40,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; /** @@ -613,62 +605,6 @@ public synchronized XdsClient returnObject(Object object) { } } - /** - * Factory for creating channels to xDS severs. - */ - abstract static class XdsChannelFactory { - @VisibleForTesting - static boolean experimentalV3SupportEnvVar = Boolean.parseBoolean( - System.getenv("GRPC_XDS_EXPERIMENTAL_V3_SUPPORT")); - - private static final String XDS_V3_SERVER_FEATURE = "xds_v3"; - private static final XdsChannelFactory DEFAULT_INSTANCE = new XdsChannelFactory() { - /** - * Creates a channel to the first server in the given list. - */ - @Override - XdsChannel createChannel(List servers) { - checkArgument(!servers.isEmpty(), "No management server provided."); - XdsLogger logger = XdsLogger.withPrefix("xds-client-channel-factory"); - ServerInfo serverInfo = servers.get(0); - String serverUri = serverInfo.getServerUri(); - logger.log(XdsLogLevel.INFO, "Creating channel to {0}", serverUri); - List channelCredsList = serverInfo.getChannelCredentials(); - ManagedChannelBuilder channelBuilder = null; - // Use the first supported channel credentials configuration. - // Currently, only "google_default" is supported. - for (ChannelCreds creds : channelCredsList) { - if (creds.getType().equals("google_default")) { - logger.log(XdsLogLevel.INFO, "Using channel credentials: google_default"); - channelBuilder = GoogleDefaultChannelBuilder.forTarget(serverUri); - break; - } - } - if (channelBuilder == null) { - logger.log(XdsLogLevel.INFO, "Using default channel credentials"); - channelBuilder = ManagedChannelBuilder.forTarget(serverUri); - } - - ManagedChannel channel = channelBuilder - .keepAliveTime(5, TimeUnit.MINUTES) - .build(); - boolean useProtocolV3 = experimentalV3SupportEnvVar - && serverInfo.getServerFeatures().contains(XDS_V3_SERVER_FEATURE); - - return new XdsChannel(channel, useProtocolV3); - } - }; - - static XdsChannelFactory getInstance() { - return DEFAULT_INSTANCE; - } - - /** - * Creates a channel to one of the provided management servers. - */ - abstract XdsChannel createChannel(List servers); - } - static final class XdsChannel { private final ManagedChannel managedChannel; private final boolean useProtocolV3; @@ -687,8 +623,4 @@ boolean isUseProtocolV3() { return useProtocolV3; } } - - interface XdsClientPoolFactory { - ObjectPool newXdsClientObjectPool(BootstrapInfo bootstrapInfo); - } } diff --git a/xds/src/main/java/io/grpc/xds/XdsClientImpl.java b/xds/src/main/java/io/grpc/xds/XdsClientImpl.java index 1dcb0c656cc..829e8711795 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientImpl.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientImpl.java @@ -53,7 +53,6 @@ import io.grpc.SynchronizationContext.ScheduledHandle; import io.grpc.internal.BackoffPolicy; import io.grpc.stub.StreamObserver; -import io.grpc.xds.Bootstrapper.ServerInfo; import io.grpc.xds.EnvoyProtoData.DropOverload; import io.grpc.xds.EnvoyProtoData.Locality; import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints; @@ -204,18 +203,14 @@ final class XdsClientImpl extends XdsClient { XdsClientImpl( String targetName, - List servers, // list of management servers - XdsChannelFactory channelFactory, + XdsChannel channel, Node node, SynchronizationContext syncContext, ScheduledExecutorService timeService, BackoffPolicy.Provider backoffPolicyProvider, Supplier stopwatchSupplier) { this.targetName = checkNotNull(targetName, "targetName"); - XdsChannel xdsChannel = - checkNotNull(channelFactory, "channelFactory") - .createChannel(checkNotNull(servers, "servers")); - this.xdsChannel = xdsChannel; + this.xdsChannel = checkNotNull(channel, "channel"); this.node = checkNotNull(node, "node"); this.syncContext = checkNotNull(syncContext, "syncContext"); this.timeService = checkNotNull(timeService, "timeService"); diff --git a/xds/src/main/java/io/grpc/xds/XdsClientWrapperForServerSds.java b/xds/src/main/java/io/grpc/xds/XdsClientWrapperForServerSds.java index e79726e8a1b..08742d419bd 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientWrapperForServerSds.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientWrapperForServerSds.java @@ -33,6 +33,7 @@ import io.grpc.xds.EnvoyServerProtoData.DownstreamTlsContext; import io.grpc.xds.EnvoyServerProtoData.FilterChain; import io.grpc.xds.EnvoyServerProtoData.FilterChainMatch; +import io.grpc.xds.XdsClient.XdsChannel; import io.netty.channel.Channel; import io.netty.channel.epoll.Epoll; import io.netty.channel.epoll.EpollEventLoopGroup; @@ -120,13 +121,14 @@ public boolean hasXdsClient() { public void createXdsClientAndStart() throws IOException { checkState(xdsClient == null, "start() called more than once"); Bootstrapper.BootstrapInfo bootstrapInfo; - List serverList; + XdsChannel channel; try { bootstrapInfo = Bootstrapper.getInstance().readBootstrap(); - serverList = bootstrapInfo.getServers(); + List serverList = bootstrapInfo.getServers(); if (serverList.isEmpty()) { throw new XdsInitializationException("No management server provided by bootstrap"); } + channel = XdsChannelFactory.getInstance().createChannel(serverList); } catch (XdsInitializationException e) { reportError(Status.fromThrowable(e)); throw new IOException(e); @@ -136,8 +138,7 @@ public void createXdsClientAndStart() throws IOException { XdsClientImpl xdsClientImpl = new XdsClientImpl( "", - serverList, - XdsClient.XdsChannelFactory.getInstance(), + channel, node, createSynchronizationContext(), timeService, diff --git a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java index f6e00cd85fb..534ac7f86fa 100644 --- a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java +++ b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java @@ -39,8 +39,9 @@ import io.grpc.xds.ThreadSafeRandom.ThreadSafeRandomImpl; import io.grpc.xds.XdsClient.ConfigUpdate; import io.grpc.xds.XdsClient.ConfigWatcher; -import io.grpc.xds.XdsClient.XdsClientPoolFactory; +import io.grpc.xds.XdsClient.XdsChannel; import io.grpc.xds.XdsLogger.XdsLogLevel; +import io.grpc.xds.XdsNameResolverProvider.XdsClientPoolFactory; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -70,6 +71,7 @@ final class XdsNameResolver extends NameResolver { private final ServiceConfigParser serviceConfigParser; private final SynchronizationContext syncContext; private final Bootstrapper bootstrapper; + private final XdsChannelFactory channelFactory; private final XdsClientPoolFactory xdsClientPoolFactory; private final ThreadSafeRandom random; private final ConcurrentMap clusterRefs = new ConcurrentHashMap<>(); @@ -85,20 +87,23 @@ final class XdsNameResolver extends NameResolver { SynchronizationContext syncContext, XdsClientPoolFactory xdsClientPoolFactory) { this(name, serviceConfigParser, syncContext, Bootstrapper.getInstance(), - xdsClientPoolFactory, ThreadSafeRandomImpl.instance); + XdsChannelFactory.getInstance(), xdsClientPoolFactory, ThreadSafeRandomImpl.instance); } + @VisibleForTesting XdsNameResolver( String name, ServiceConfigParser serviceConfigParser, SynchronizationContext syncContext, Bootstrapper bootstrapper, + XdsChannelFactory channelFactory, XdsClientPoolFactory xdsClientPoolFactory, ThreadSafeRandom random) { authority = GrpcUtil.checkAuthority(checkNotNull(name, "name")); this.serviceConfigParser = checkNotNull(serviceConfigParser, "serviceConfigParser"); this.syncContext = checkNotNull(syncContext, "syncContext"); this.bootstrapper = checkNotNull(bootstrapper, "bootstrapper"); + this.channelFactory = checkNotNull(channelFactory, "channelFactory"); this.xdsClientPoolFactory = checkNotNull(xdsClientPoolFactory, "xdsClientPoolFactory"); this.random = checkNotNull(random, "random"); logger = XdsLogger.withLogId(InternalLogId.allocate("xds-resolver", name)); @@ -114,14 +119,16 @@ public String getServiceAuthority() { public void start(Listener2 listener) { this.listener = checkNotNull(listener, "listener"); BootstrapInfo bootstrapInfo; + XdsChannel channel; try { bootstrapInfo = bootstrapper.readBootstrap(); + channel = channelFactory.createChannel(bootstrapInfo.getServers()); } catch (Exception e) { listener.onError( - Status.UNAVAILABLE.withDescription("Failed to load xDS bootstrap").withCause(e)); + Status.UNAVAILABLE.withDescription("Failed to initialize xDS").withCause(e)); return; } - xdsClientPool = xdsClientPoolFactory.newXdsClientObjectPool(bootstrapInfo); + xdsClientPool = xdsClientPoolFactory.newXdsClientObjectPool(bootstrapInfo, channel); xdsClient = xdsClientPool.getObject(); xdsClient.watchConfigData(authority, new ConfigWatcherImpl()); } diff --git a/xds/src/main/java/io/grpc/xds/XdsNameResolverProvider.java b/xds/src/main/java/io/grpc/xds/XdsNameResolverProvider.java index d00929cc38f..ee7c1d68a01 100644 --- a/xds/src/main/java/io/grpc/xds/XdsNameResolverProvider.java +++ b/xds/src/main/java/io/grpc/xds/XdsNameResolverProvider.java @@ -31,9 +31,8 @@ import io.grpc.internal.ObjectPool; import io.grpc.xds.Bootstrapper.BootstrapInfo; import io.grpc.xds.XdsClient.RefCountedXdsClientObjectPool; -import io.grpc.xds.XdsClient.XdsChannelFactory; +import io.grpc.xds.XdsClient.XdsChannel; import io.grpc.xds.XdsClient.XdsClientFactory; -import io.grpc.xds.XdsClient.XdsClientPoolFactory; import java.net.URI; import java.util.concurrent.ScheduledExecutorService; @@ -64,11 +63,8 @@ public XdsNameResolver newNameResolver(URI targetUri, Args args) { String name = targetPath.substring(1); XdsClientPoolFactory xdsClientPoolFactory = new RefCountedXdsClientPoolFactory( - name, - XdsChannelFactory.getInstance(), - args.getSynchronizationContext(), args.getScheduledExecutorService(), - new ExponentialBackoffPolicy.Provider(), - GrpcUtil.STOPWATCH_SUPPLIER); + name, args.getSynchronizationContext(), args.getScheduledExecutorService(), + new ExponentialBackoffPolicy.Provider(), GrpcUtil.STOPWATCH_SUPPLIER); return new XdsNameResolver( name, args.getServiceConfigParser(), args.getSynchronizationContext(), xdsClientPoolFactory); @@ -95,7 +91,6 @@ protected int priority() { static class RefCountedXdsClientPoolFactory implements XdsClientPoolFactory { private final String serviceName; - private final XdsChannelFactory channelFactory; private final SynchronizationContext syncContext; private final ScheduledExecutorService timeService; private final BackoffPolicy.Provider backoffPolicyProvider; @@ -103,13 +98,11 @@ static class RefCountedXdsClientPoolFactory implements XdsClientPoolFactory { RefCountedXdsClientPoolFactory( String serviceName, - XdsChannelFactory channelFactory, SynchronizationContext syncContext, ScheduledExecutorService timeService, BackoffPolicy.Provider backoffPolicyProvider, Supplier stopwatchSupplier) { this.serviceName = checkNotNull(serviceName, "serviceName"); - this.channelFactory = checkNotNull(channelFactory, "channelFactory"); this.syncContext = checkNotNull(syncContext, "syncContext"); this.timeService = checkNotNull(timeService, "timeService"); this.backoffPolicyProvider = checkNotNull(backoffPolicyProvider, "backoffPolicyProvider"); @@ -117,16 +110,21 @@ static class RefCountedXdsClientPoolFactory implements XdsClientPoolFactory { } @Override - public ObjectPool newXdsClientObjectPool(final BootstrapInfo bootstrapInfo) { + public ObjectPool newXdsClientObjectPool( + final BootstrapInfo bootstrapInfo, final XdsChannel channel) { XdsClientFactory xdsClientFactory = new XdsClientFactory() { @Override XdsClient createXdsClient() { return new XdsClientImpl( - serviceName, bootstrapInfo.getServers(), channelFactory, bootstrapInfo.getNode(), - syncContext, timeService, backoffPolicyProvider, stopwatchSupplier); + serviceName, channel, bootstrapInfo.getNode(), syncContext, timeService, + backoffPolicyProvider, stopwatchSupplier); } }; return new RefCountedXdsClientObjectPool(xdsClientFactory); } } + + interface XdsClientPoolFactory { + ObjectPool newXdsClientObjectPool(BootstrapInfo bootstrapInfo, XdsChannel channel); + } } diff --git a/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java index 7461159eaa9..192201e7a94 100644 --- a/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java @@ -80,7 +80,6 @@ import io.grpc.xds.LocalityStore.LocalityStoreFactory; import io.grpc.xds.XdsClient.EndpointUpdate; import io.grpc.xds.XdsClient.XdsChannel; -import io.grpc.xds.XdsClient.XdsChannelFactory; import java.net.InetSocketAddress; import java.util.ArrayDeque; import java.util.ArrayList; @@ -240,8 +239,7 @@ public StreamObserver streamAggregatedResources( xdsClientPoolFromResolveAddresses = new FakeXdsClientPool( new XdsClientImpl( SERVICE_AUTHORITY, - serverList, - channelFactory, + new XdsChannel(channel, /* useProtocolV3= */ false), node, syncContext, fakeClock.getScheduledExecutorService(), diff --git a/xds/src/test/java/io/grpc/xds/XdsChannelFactoryTest.java b/xds/src/test/java/io/grpc/xds/XdsChannelFactoryTest.java new file mode 100644 index 00000000000..090f6ab94ec --- /dev/null +++ b/xds/src/test/java/io/grpc/xds/XdsChannelFactoryTest.java @@ -0,0 +1,99 @@ +/* + * Copyright 2020 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.xds; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.fail; + +import io.grpc.xds.Bootstrapper.ChannelCreds; +import io.grpc.xds.Bootstrapper.ServerInfo; +import io.grpc.xds.XdsClient.XdsChannel; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for {@link XdsChannelFactory}. + */ +@RunWith(JUnit4.class) +public class XdsChannelFactoryTest { + + private final XdsChannelFactory channelFactory = XdsChannelFactory.getInstance(); + private final List channels = new ArrayList<>(); + private ServerInfo server1; // google_default + private ServerInfo server2; // plaintext, v3 + private ServerInfo server3; // unsupported + + @Before + public void setUp() { + ChannelCreds googleDefault = new ChannelCreds("google_default", null); + ChannelCreds insecure = new ChannelCreds("insecure", null); + ChannelCreds unsupported = new ChannelCreds("unsupported", null); + server1 = new ServerInfo("server1.com", Collections.singletonList(googleDefault), + Collections.emptyList()); + server2 = new ServerInfo("server2.com", Collections.singletonList(insecure), + Collections.singletonList("xds_v3")); + server3 = new ServerInfo("server4.com", Collections.singletonList(unsupported), + Collections.emptyList()); + } + + @After + public void tearDown() { + for (XdsChannel channel : channels) { + channel.getManagedChannel().shutdown(); + } + } + + @Test + public void failToCreateChannel_unsupportedChannelCreds() { + try { + createChannel(server3); + fail("Should have thrown"); + } catch (XdsInitializationException expected) { + } + } + + @Test + public void defaultUseV2ProtocolL() throws XdsInitializationException { + XdsChannel channel = createChannel(server1); + assertThat(channel.isUseProtocolV3()).isFalse(); + } + + @Test + public void supportServerFeature_v3Protocol() throws XdsInitializationException { + boolean originalV3SupportEnvVar = XdsChannelFactory.experimentalV3SupportEnvVar; + XdsChannelFactory.experimentalV3SupportEnvVar = true; + try { + XdsChannel channel = createChannel(server2); + assertThat(channel.isUseProtocolV3()).isTrue(); + } finally { + XdsChannelFactory.experimentalV3SupportEnvVar = originalV3SupportEnvVar; + } + } + + private XdsChannel createChannel(ServerInfo... servers) throws XdsInitializationException { + XdsChannel channel = channelFactory.createChannel(Arrays.asList(servers)); + channels.add(channel); + return channel; + } +} diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java index a9b8e42b6bb..f75b67fccd2 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java @@ -86,8 +86,6 @@ import io.grpc.internal.FakeClock.TaskFilter; import io.grpc.stub.StreamObserver; import io.grpc.testing.GrpcCleanupRule; -import io.grpc.xds.Bootstrapper.ChannelCreds; -import io.grpc.xds.Bootstrapper.ServerInfo; import io.grpc.xds.EnvoyProtoData.DropOverload; import io.grpc.xds.EnvoyProtoData.LbEndpoint; import io.grpc.xds.EnvoyProtoData.Locality; @@ -100,7 +98,6 @@ import io.grpc.xds.XdsClient.EndpointUpdate; import io.grpc.xds.XdsClient.EndpointWatcher; import io.grpc.xds.XdsClient.XdsChannel; -import io.grpc.xds.XdsClient.XdsChannelFactory; import io.grpc.xds.XdsClientImpl.MessagePrinter; import java.io.IOException; import java.util.ArrayDeque; @@ -279,23 +276,10 @@ public void cancelled(Context context) { channel = cleanupRule.register(InProcessChannelBuilder.forName(serverName).directExecutor().build()); - List servers = - ImmutableList.of(new ServerInfo(serverName, ImmutableList.of(), null)); - XdsChannelFactory channelFactory = new XdsChannelFactory() { - @Override - XdsChannel createChannel(List servers) { - ServerInfo serverInfo = Iterables.getOnlyElement(servers); - assertThat(serverInfo.getServerUri()).isEqualTo(serverName); - assertThat(serverInfo.getChannelCredentials()).isEmpty(); - return new XdsChannel(channel, /* useProtocolV3= */ true); - } - }; - xdsClient = new XdsClientImpl( TARGET_AUTHORITY, - servers, - channelFactory, + new XdsChannel(channel, /* useProtocolV3= */ true), EnvoyProtoData.Node.newBuilder().build(), syncContext, fakeClock.getScheduledExecutorService(), diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTestForListener.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTestForListener.java index 77c073cf7b6..c4028eec348 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientImplTestForListener.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTestForListener.java @@ -64,15 +64,12 @@ import io.grpc.internal.FakeClock.TaskFilter; import io.grpc.stub.StreamObserver; import io.grpc.testing.GrpcCleanupRule; -import io.grpc.xds.Bootstrapper.ChannelCreds; -import io.grpc.xds.Bootstrapper.ServerInfo; import io.grpc.xds.EnvoyProtoData.Address; import io.grpc.xds.EnvoyProtoData.Node; import io.grpc.xds.XdsClient.ConfigWatcher; import io.grpc.xds.XdsClient.ListenerUpdate; import io.grpc.xds.XdsClient.ListenerWatcher; import io.grpc.xds.XdsClient.XdsChannel; -import io.grpc.xds.XdsClient.XdsChannelFactory; import io.grpc.xds.internal.sds.CommonTlsContextTestsUtil; import java.io.IOException; import java.util.ArrayDeque; @@ -200,21 +197,9 @@ public void cancelled(Context context) { channel = cleanupRule.register(InProcessChannelBuilder.forName(serverName).directExecutor().build()); - List servers = - ImmutableList.of(new ServerInfo(serverName, ImmutableList.of(), null)); - XdsChannelFactory channelFactory = new XdsChannelFactory() { - @Override - XdsChannel createChannel(List servers) { - ServerInfo serverInfo = Iterables.getOnlyElement(servers); - assertThat(serverInfo.getServerUri()).isEqualTo(serverName); - assertThat(serverInfo.getChannelCredentials()).isEmpty(); - return new XdsChannel(channel, false); - } - }; - xdsClient = - new XdsClientImpl("", servers, channelFactory, NODE, syncContext, - fakeClock.getScheduledExecutorService(), backoffPolicyProvider, + new XdsClientImpl("", new XdsChannel(channel, /* useProtocolV3= */ false), NODE, + syncContext, fakeClock.getScheduledExecutorService(), backoffPolicyProvider, fakeClock.getStopwatchSupplier()); // Only the connection to management server is established, no RPC request is sent until at // least one watcher is registered. diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java index 51b5b5db684..17351d00d17 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java @@ -86,8 +86,6 @@ import io.grpc.internal.FakeClock.TaskFilter; import io.grpc.stub.StreamObserver; import io.grpc.testing.GrpcCleanupRule; -import io.grpc.xds.Bootstrapper.ChannelCreds; -import io.grpc.xds.Bootstrapper.ServerInfo; import io.grpc.xds.EnvoyProtoData.DropOverload; import io.grpc.xds.EnvoyProtoData.LbEndpoint; import io.grpc.xds.EnvoyProtoData.Locality; @@ -100,7 +98,6 @@ import io.grpc.xds.XdsClient.EndpointUpdate; import io.grpc.xds.XdsClient.EndpointWatcher; import io.grpc.xds.XdsClient.XdsChannel; -import io.grpc.xds.XdsClient.XdsChannelFactory; import io.grpc.xds.XdsClientImpl.MessagePrinter; import java.io.IOException; import java.util.ArrayDeque; @@ -278,23 +275,10 @@ public void cancelled(Context context) { channel = cleanupRule.register(InProcessChannelBuilder.forName(serverName).directExecutor().build()); - List servers = - ImmutableList.of(new ServerInfo(serverName, ImmutableList.of(), null)); - XdsChannelFactory channelFactory = new XdsChannelFactory() { - @Override - XdsChannel createChannel(List servers) { - ServerInfo serverInfo = Iterables.getOnlyElement(servers); - assertThat(serverInfo.getServerUri()).isEqualTo(serverName); - assertThat(serverInfo.getChannelCredentials()).isEmpty(); - return new XdsChannel(channel, false); - } - }; - xdsClient = new XdsClientImpl( TARGET_AUTHORITY, - servers, - channelFactory, + new XdsChannel(channel, /* useProtocolV3= */ false), Node.newBuilder().build(), syncContext, fakeClock.getScheduledExecutorService(), diff --git a/xds/src/test/java/io/grpc/xds/XdsClientTest.java b/xds/src/test/java/io/grpc/xds/XdsClientTest.java index 9278a80dbb5..56fca6dc09b 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientTest.java @@ -21,12 +21,7 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; -import com.google.common.collect.ImmutableList; -import io.grpc.xds.Bootstrapper.ChannelCreds; -import io.grpc.xds.Bootstrapper.ServerInfo; import io.grpc.xds.XdsClient.RefCountedXdsClientObjectPool; -import io.grpc.xds.XdsClient.XdsChannel; -import io.grpc.xds.XdsClient.XdsChannelFactory; import io.grpc.xds.XdsClient.XdsClientFactory; import org.junit.Rule; import org.junit.Test; @@ -107,58 +102,4 @@ XdsClient createXdsClient() { XdsClient xdsClient2 = xdsClientPool.getObject(); assertThat(xdsClient2).isNotSameInstanceAs(xdsClient1); } - - @Test - public void channelFactorySupportsV3() { - boolean originalV3SupportEnvVar = XdsChannelFactory.experimentalV3SupportEnvVar; - try { - XdsChannelFactory xdsChannelFactory = XdsChannelFactory.getInstance(); - XdsChannelFactory.experimentalV3SupportEnvVar = true; - XdsChannel xdsChannel = - xdsChannelFactory.createChannel( - ImmutableList.of( - new ServerInfo( - "xdsserver.com", - ImmutableList.of(), - ImmutableList.of()), - new ServerInfo( - "xdsserver2.com", - ImmutableList.of(), - ImmutableList.of("xds_v3")))); - xdsChannel.getManagedChannel().shutdown(); - assertThat(xdsChannel.isUseProtocolV3()).isFalse(); - - XdsChannelFactory.experimentalV3SupportEnvVar = false; - xdsChannel = - xdsChannelFactory.createChannel( - ImmutableList.of( - new ServerInfo( - "xdsserver.com", - ImmutableList.of(), - ImmutableList.of("xds_v3")), - new ServerInfo( - "xdsserver2.com", - ImmutableList.of(), - ImmutableList.of("baz")))); - xdsChannel.getManagedChannel().shutdown(); - assertThat(xdsChannel.isUseProtocolV3()).isFalse(); - - XdsChannelFactory.experimentalV3SupportEnvVar = true; - xdsChannel = - xdsChannelFactory.createChannel( - ImmutableList.of( - new ServerInfo( - "xdsserver.com", - ImmutableList.of(), - ImmutableList.of("xds_v3")), - new ServerInfo( - "xdsserver2.com", - ImmutableList.of(), - ImmutableList.of("baz")))); - xdsChannel.getManagedChannel().shutdown(); - assertThat(xdsChannel.isUseProtocolV3()).isTrue(); - } finally { - XdsChannelFactory.experimentalV3SupportEnvVar = originalV3SupportEnvVar; - } - } } diff --git a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java index 30658a045fe..797bd45a561 100644 --- a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java @@ -18,6 +18,7 @@ import static com.google.common.truth.Truth.assertThat; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -30,6 +31,7 @@ import io.grpc.CallOptions; import io.grpc.InternalConfigSelector; import io.grpc.InternalConfigSelector.Result; +import io.grpc.ManagedChannel; import io.grpc.Metadata; import io.grpc.MethodDescriptor; import io.grpc.MethodDescriptor.MethodType; @@ -46,11 +48,13 @@ import io.grpc.internal.PickSubchannelArgsImpl; import io.grpc.testing.TestMethodDescriptors; import io.grpc.xds.Bootstrapper.BootstrapInfo; +import io.grpc.xds.Bootstrapper.ServerInfo; import io.grpc.xds.EnvoyProtoData.ClusterWeight; import io.grpc.xds.EnvoyProtoData.Node; import io.grpc.xds.EnvoyProtoData.Route; import io.grpc.xds.EnvoyProtoData.RouteAction; -import io.grpc.xds.XdsClient.XdsClientPoolFactory; +import io.grpc.xds.XdsClient.XdsChannel; +import io.grpc.xds.XdsNameResolverProvider.XdsClientPoolFactory; import java.io.IOException; import java.util.Arrays; import java.util.Collections; @@ -88,6 +92,12 @@ public ConfigOrError parseServiceConfig(Map rawServiceConfig) { return ConfigOrError.fromConfig(rawServiceConfig); } }; + private final XdsChannelFactory channelFactory = new XdsChannelFactory() { + @Override + XdsChannel createChannel(List servers) throws XdsInitializationException { + return new XdsChannel(mock(ManagedChannel.class), false); + } + }; private final FakeXdsClientPoolFactory xdsClientPoolFactory = new FakeXdsClientPoolFactory(); private final String cluster1 = "cluster-foo.googleapis.com"; private final String cluster2 = "cluster-bar.googleapis.com"; @@ -119,7 +129,7 @@ public BootstrapInfo readBootstrap() { } }; resolver = new XdsNameResolver(AUTHORITY, serviceConfigParser, syncContext, bootstrapper, - xdsClientPoolFactory, mockRandom); + channelFactory, xdsClientPoolFactory, mockRandom); } @Test @@ -131,15 +141,46 @@ public BootstrapInfo readBootstrap() throws XdsInitializationException { } }; resolver = new XdsNameResolver(AUTHORITY, serviceConfigParser, syncContext, bootstrapper, - xdsClientPoolFactory, mockRandom); + channelFactory, xdsClientPoolFactory, mockRandom); resolver.start(mockListener); verify(mockListener).onError(errorCaptor.capture()); Status error = errorCaptor.getValue(); assertThat(error.getCode()).isEqualTo(Code.UNAVAILABLE); - assertThat(error.getDescription()).isEqualTo("Failed to load xDS bootstrap"); + assertThat(error.getDescription()).isEqualTo("Failed to initialize xDS"); assertThat(error.getCause()).hasMessageThat().isEqualTo("Fail to read bootstrap file"); } + @Test + public void resolve_failToCreateXdsChannel() { + Bootstrapper bootstrapper = new Bootstrapper() { + @Override + public BootstrapInfo readBootstrap() { + return new BootstrapInfo( + ImmutableList.of( + new ServerInfo( + "trafficdirector.googleapis.com", + ImmutableList.of(), ImmutableList.of())), + Node.newBuilder().build(), + null); + } + }; + XdsChannelFactory channelFactory = new XdsChannelFactory() { + @Override + XdsChannel createChannel(List servers) throws XdsInitializationException { + throw new XdsInitializationException("No server with supported channel creds found"); + } + }; + resolver = new XdsNameResolver(AUTHORITY, serviceConfigParser, syncContext, bootstrapper, + channelFactory, xdsClientPoolFactory, mockRandom); + resolver.start(mockListener); + verify(mockListener).onError(errorCaptor.capture()); + Status error = errorCaptor.getValue(); + assertThat(error.getCode()).isEqualTo(Code.UNAVAILABLE); + assertThat(error.getDescription()).isEqualTo("Failed to initialize xDS"); + assertThat(error.getCause()).hasMessageThat() + .isEqualTo("No server with supported channel creds found"); + } + @SuppressWarnings("unchecked") @Test public void resolve_resourceNotFound() { @@ -472,7 +513,8 @@ public void generateServiceConfig_forMethodTimeoutConfig() throws IOException { private final class FakeXdsClientPoolFactory implements XdsClientPoolFactory { @Override - public ObjectPool newXdsClientObjectPool(BootstrapInfo bootstrapInfo) { + public ObjectPool newXdsClientObjectPool( + BootstrapInfo bootstrapInfo, XdsChannel channel) { return new ObjectPool() { @Override public XdsClient getObject() { From da100e8e49a9e9500e25f8b20d7981b896aca47a Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Mon, 21 Sep 2020 10:42:06 -0700 Subject: [PATCH 55/86] build, examples, README.md: Update protobuf gradle plugin version to 0.8.13 (#7355) Updated protobuf gradle plugin version to 0.8.13. Fixed Android Kokoro's memory issue by forcing to use a new Gradle daemon for building the previous commit. --- README.md | 2 +- buildscripts/kokoro/android.sh | 3 ++- examples/android/clientcache/build.gradle | 2 +- examples/android/helloworld/build.gradle | 2 +- examples/android/routeguide/build.gradle | 2 +- examples/android/strictmode/build.gradle | 2 +- examples/build.gradle | 4 ++-- examples/example-alts/build.gradle | 4 ++-- examples/example-gauth/build.gradle | 4 ++-- examples/example-jwt-auth/build.gradle | 4 ++-- examples/example-tls/build.gradle | 4 ++-- settings.gradle | 2 +- 12 files changed, 18 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index b1a66311d75..737fc40c1d6 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ For protobuf-based codegen integrated with the Gradle build system, you can use [protobuf-gradle-plugin][]: ```gradle plugins { - id 'com.google.protobuf' version '0.8.8' + id 'com.google.protobuf' version '0.8.13' } protobuf { diff --git a/buildscripts/kokoro/android.sh b/buildscripts/kokoro/android.sh index 951488f747d..33f3f7b20bf 100755 --- a/buildscripts/kokoro/android.sh +++ b/buildscripts/kokoro/android.sh @@ -89,8 +89,9 @@ new_apk_size="$(stat --printf=%s $HELLO_WORLD_OUTPUT_DIR/apk/release/app-release # Get the APK size and dex count stats using the pull request base commit cd $BASE_DIR/github/grpc-java -git checkout HEAD^ ./gradlew clean +git checkout HEAD^ +./gradlew --stop # use a new daemon to build the previous commit ./gradlew publishToMavenLocal cd examples/android/helloworld/ ../../gradlew build diff --git a/examples/android/clientcache/build.gradle b/examples/android/clientcache/build.gradle index 436383e2270..60883b8037c 100644 --- a/examples/android/clientcache/build.gradle +++ b/examples/android/clientcache/build.gradle @@ -7,7 +7,7 @@ buildscript { } dependencies { classpath 'com.android.tools.build:gradle:4.0.0' - classpath "com.google.protobuf:protobuf-gradle-plugin:0.8.8" + classpath "com.google.protobuf:protobuf-gradle-plugin:0.8.13" // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files diff --git a/examples/android/helloworld/build.gradle b/examples/android/helloworld/build.gradle index 436383e2270..60883b8037c 100644 --- a/examples/android/helloworld/build.gradle +++ b/examples/android/helloworld/build.gradle @@ -7,7 +7,7 @@ buildscript { } dependencies { classpath 'com.android.tools.build:gradle:4.0.0' - classpath "com.google.protobuf:protobuf-gradle-plugin:0.8.8" + classpath "com.google.protobuf:protobuf-gradle-plugin:0.8.13" // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files diff --git a/examples/android/routeguide/build.gradle b/examples/android/routeguide/build.gradle index cafba46c04b..4f1c7873371 100644 --- a/examples/android/routeguide/build.gradle +++ b/examples/android/routeguide/build.gradle @@ -7,7 +7,7 @@ buildscript { } dependencies { classpath 'com.android.tools.build:gradle:4.0.0' - classpath "com.google.protobuf:protobuf-gradle-plugin:0.8.8" + classpath "com.google.protobuf:protobuf-gradle-plugin:0.8.13" // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files diff --git a/examples/android/strictmode/build.gradle b/examples/android/strictmode/build.gradle index 436383e2270..60883b8037c 100644 --- a/examples/android/strictmode/build.gradle +++ b/examples/android/strictmode/build.gradle @@ -7,7 +7,7 @@ buildscript { } dependencies { classpath 'com.android.tools.build:gradle:4.0.0' - classpath "com.google.protobuf:protobuf-gradle-plugin:0.8.8" + classpath "com.google.protobuf:protobuf-gradle-plugin:0.8.13" // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files diff --git a/examples/build.gradle b/examples/build.gradle index 68bba439d65..aacb920e557 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -1,8 +1,8 @@ plugins { // Provide convenience executables for trying out the examples. id 'application' - // ASSUMES GRADLE 2.12 OR HIGHER. Use plugin version 0.7.5 with earlier gradle versions - id 'com.google.protobuf' version '0.8.8' + // ASSUMES GRADLE 5.6 OR HIGHER. Use plugin version 0.8.10 with earlier gradle versions + id 'com.google.protobuf' version '0.8.13' // Generate IntelliJ IDEA's .idea & .iml project files id 'idea' } diff --git a/examples/example-alts/build.gradle b/examples/example-alts/build.gradle index 7b34c2da0e2..9e71cdb9f50 100644 --- a/examples/example-alts/build.gradle +++ b/examples/example-alts/build.gradle @@ -1,8 +1,8 @@ plugins { // Provide convenience executables for trying out the examples. id 'application' - // ASSUMES GRADLE 2.12 OR HIGHER. Use plugin version 0.7.5 with earlier gradle versions - id 'com.google.protobuf' version '0.8.8' + // ASSUMES GRADLE 5.6 OR HIGHER. Use plugin version 0.8.10 with earlier gradle versions + id 'com.google.protobuf' version '0.8.13' // Generate IntelliJ IDEA's .idea & .iml project files id 'idea' } diff --git a/examples/example-gauth/build.gradle b/examples/example-gauth/build.gradle index 0a2a92d8fac..5283b4a19bc 100644 --- a/examples/example-gauth/build.gradle +++ b/examples/example-gauth/build.gradle @@ -1,8 +1,8 @@ plugins { // Provide convenience executables for trying out the examples. id 'application' - // ASSUMES GRADLE 2.12 OR HIGHER. Use plugin version 0.7.5 with earlier gradle versions - id 'com.google.protobuf' version '0.8.8' + // ASSUMES GRADLE 5.6 OR HIGHER. Use plugin version 0.8.10 with earlier gradle versions + id 'com.google.protobuf' version '0.8.13' // Generate IntelliJ IDEA's .idea & .iml project files id 'idea' } diff --git a/examples/example-jwt-auth/build.gradle b/examples/example-jwt-auth/build.gradle index 588daf3794a..a258c98c484 100644 --- a/examples/example-jwt-auth/build.gradle +++ b/examples/example-jwt-auth/build.gradle @@ -1,8 +1,8 @@ plugins { // Provide convenience executables for trying out the examples. id 'application' - // ASSUMES GRADLE 2.12 OR HIGHER. Use plugin version 0.7.5 with earlier gradle versions - id 'com.google.protobuf' version '0.8.8' + // ASSUMES GRADLE 5.6 OR HIGHER. Use plugin version 0.8.10 with earlier gradle versions + id 'com.google.protobuf' version '0.8.13' // Generate IntelliJ IDEA's .idea & .iml project files id 'idea' } diff --git a/examples/example-tls/build.gradle b/examples/example-tls/build.gradle index fbec7406092..bebb2946ce2 100644 --- a/examples/example-tls/build.gradle +++ b/examples/example-tls/build.gradle @@ -1,8 +1,8 @@ plugins { // Provide convenience executables for trying out the examples. id 'application' - // ASSUMES GRADLE 2.12 OR HIGHER. Use plugin version 0.7.5 with earlier gradle versions - id 'com.google.protobuf' version '0.8.8' + // ASSUMES GRADLE 5.6 OR HIGHER. Use plugin version 0.8.10 with earlier gradle versions + id 'com.google.protobuf' version '0.8.13' // Generate IntelliJ IDEA's .idea & .iml project files id 'idea' } diff --git a/settings.gradle b/settings.gradle index 9592c7eda93..0b75e7f3859 100644 --- a/settings.gradle +++ b/settings.gradle @@ -5,7 +5,7 @@ pluginManagement { id "com.github.johnrengelman.shadow" version "2.0.4" id "com.github.kt3k.coveralls" version "2.0.1" id "com.google.osdetector" version "1.4.0" - id "com.google.protobuf" version "0.8.12" + id "com.google.protobuf" version "0.8.13" id "digital.wup.android-maven-publish" version "3.6.2" id "me.champeau.gradle.japicmp" version "0.2.5" id "me.champeau.gradle.jmh" version "0.5.0" From d333304a2b7d87d1300a7f2a81ab19f8ddea780e Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Mon, 21 Sep 2020 16:44:09 -0700 Subject: [PATCH 56/86] xds: promote EdsLoadBalancer2 (#7444) Effectively migrate to the new codepath of hierarchical LB policies. --- xds/src/main/java/io/grpc/xds/EdsLoadBalancerProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xds/src/main/java/io/grpc/xds/EdsLoadBalancerProvider.java b/xds/src/main/java/io/grpc/xds/EdsLoadBalancerProvider.java index 2a8acc8afab..fb44cecf054 100644 --- a/xds/src/main/java/io/grpc/xds/EdsLoadBalancerProvider.java +++ b/xds/src/main/java/io/grpc/xds/EdsLoadBalancerProvider.java @@ -56,7 +56,7 @@ public String getPolicyName() { @Override public LoadBalancer newLoadBalancer(Helper helper) { - return new EdsLoadBalancer(helper); + return new EdsLoadBalancer2(helper); } @Override From a5ae55e984a1ef88bafa732cc6c4dc401db9f632 Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Mon, 21 Sep 2020 18:16:28 -0700 Subject: [PATCH 57/86] xds: sync Envoy proto to commit fd28e42f31730f5ed6f13f52999692a4885dd312 (2020-09-15) (#7445) Sync Envoy proto to commit fd28e42f31730f5ed6f13f52999692a4885dd312 (2020-09-15, internal Envoy import CL: cl/332279092). Suppressed warning for using listening_addresses in Node and added TODO for cleaning it up. --- .../main/java/io/grpc/xds/EnvoyProtoData.java | 2 + .../main/java/io/grpc/xds/XdsClientImpl.java | 1 + .../java/io/grpc/xds/EnvoyProtoDataTest.java | 1 + xds/third_party/envoy/import.sh | 5 +- .../src/main/proto/envoy/api/v2/cluster.proto | 2 +- .../envoy/api/v2/route/route_components.proto | 2 +- .../envoy/config/accesslog/v3/accesslog.proto | 84 ++++++--- .../envoy/config/cluster/v3/cluster.proto | 94 +++++++++- .../proto/envoy/config/core/v3/address.proto | 14 ++ .../proto/envoy/config/core/v3/base.proto | 4 +- .../envoy/config/core/v3/grpc_service.proto | 6 + .../proto/envoy/config/core/v3/protocol.proto | 20 ++- .../core/v3/substitution_format_string.proto | 7 + .../envoy/config/listener/v3/listener.proto | 19 ++- .../listener/v3/listener_components.proto | 23 ++- .../config/route/v3/route_components.proto | 160 +++++++++++++++++- .../envoy/config/route/v3/scoped_route.proto | 3 + .../v3/http_connection_manager.proto | 6 +- .../transport_sockets/tls/v3/common.proto | 11 +- .../transport_sockets/tls/v3/tls.proto | 6 +- .../envoy/type/matcher/v3/metadata.proto | 104 ++++++++++++ .../proto/envoy/type/matcher/v3/number.proto | 32 ++++ .../proto/envoy/type/matcher/v3/string.proto | 10 +- .../proto/envoy/type/matcher/v3/value.proto | 71 ++++++++ 24 files changed, 633 insertions(+), 54 deletions(-) create mode 100644 xds/third_party/envoy/src/main/proto/envoy/type/matcher/v3/metadata.proto create mode 100644 xds/third_party/envoy/src/main/proto/envoy/type/matcher/v3/number.proto create mode 100644 xds/third_party/envoy/src/main/proto/envoy/type/matcher/v3/value.proto diff --git a/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java b/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java index ad78a6967c5..2516ab17ca2 100644 --- a/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java +++ b/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java @@ -226,6 +226,7 @@ static final class Builder { private Map metadata; @Nullable private Locality locality; + // TODO(sanjaypujare): eliminate usage of listening_addresses field. private final List

listeningAddresses = new ArrayList<>(); private String buildVersion = ""; private String userAgentName = ""; @@ -326,6 +327,7 @@ List
getListeningAddresses() { return listeningAddresses; } + @SuppressWarnings("deprecation") @VisibleForTesting public io.envoyproxy.envoy.config.core.v3.Node toEnvoyProtoNode() { io.envoyproxy.envoy.config.core.v3.Node.Builder builder = diff --git a/xds/src/main/java/io/grpc/xds/XdsClientImpl.java b/xds/src/main/java/io/grpc/xds/XdsClientImpl.java index 829e8711795..b427cdd7c90 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientImpl.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientImpl.java @@ -462,6 +462,7 @@ private void updateNodeMetadataForListenerRequest(int port) { newMetadata.putAll(node.getMetadata()); } newMetadata.put("TRAFFICDIRECTOR_PROXYLESS", "1"); + // TODO(sanjaypujare): eliminate usage of listening_addresses. EnvoyProtoData.Address listeningAddress = new EnvoyProtoData.Address("0.0.0.0", port); node = diff --git a/xds/src/test/java/io/grpc/xds/EnvoyProtoDataTest.java b/xds/src/test/java/io/grpc/xds/EnvoyProtoDataTest.java index 2e307d6c7eb..5ffabdf3642 100644 --- a/xds/src/test/java/io/grpc/xds/EnvoyProtoDataTest.java +++ b/xds/src/test/java/io/grpc/xds/EnvoyProtoDataTest.java @@ -96,6 +96,7 @@ public void locality_equal() { .testEquals(); } + @SuppressWarnings("deprecation") @Test public void convertNode() { Node node = Node.newBuilder() diff --git a/xds/third_party/envoy/import.sh b/xds/third_party/envoy/import.sh index 80352ca520a..593bec256c6 100755 --- a/xds/third_party/envoy/import.sh +++ b/xds/third_party/envoy/import.sh @@ -18,7 +18,7 @@ set -e BRANCH=master # import VERSION from one of the google internal CLs -VERSION=3dedf1693f45239c670c5ba7598db44ff2e32c2f +VERSION=fd28e42f31730f5ed6f13f52999692a4885dd312 GIT_REPO="https://github.com/envoyproxy/envoy.git" GIT_BASE_DIR=envoy SOURCE_PROTO_BASE_DIR=envoy/api @@ -130,8 +130,11 @@ envoy/type/matcher/metadata.proto envoy/type/matcher/path.proto envoy/type/matcher/value.proto envoy/type/matcher/number.proto +envoy/type/matcher/v3/metadata.proto +envoy/type/matcher/v3/number.proto envoy/type/matcher/v3/regex.proto envoy/type/matcher/v3/string.proto +envoy/type/matcher/v3/value.proto envoy/type/metadata/v2/metadata.proto envoy/type/metadata/v3/metadata.proto envoy/type/percent.proto diff --git a/xds/third_party/envoy/src/main/proto/envoy/api/v2/cluster.proto b/xds/third_party/envoy/src/main/proto/envoy/api/v2/cluster.proto index c95de62c128..d1a50fbdb91 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/api/v2/cluster.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/api/v2/cluster.proto @@ -448,7 +448,7 @@ message Cluster { // connections to upstream hosts whenever hosts are added or removed from the cluster. bool close_connections_on_host_set_change = 6; - //Common Configuration for all consistent hashing load balancers (MaglevLb, RingHashLb, etc.) + // Common Configuration for all consistent hashing load balancers (MaglevLb, RingHashLb, etc.) ConsistentHashingLbConfig consistent_hashing_lb_config = 7; } diff --git a/xds/third_party/envoy/src/main/proto/envoy/api/v2/route/route_components.proto b/xds/third_party/envoy/src/main/proto/envoy/api/v2/route/route_components.proto index 007f71d57cb..339c7bcbc53 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/api/v2/route/route_components.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/api/v2/route/route_components.proto @@ -436,7 +436,7 @@ message RouteMatch { type.matcher.RegexMatcher safe_regex = 10 [(validate.rules).message = {required: true}]; } - // Indicates that prefix/path matching should be case insensitive. The default + // Indicates that prefix/path matching should be case sensitive. The default // is true. google.protobuf.BoolValue case_sensitive = 4; diff --git a/xds/third_party/envoy/src/main/proto/envoy/config/accesslog/v3/accesslog.proto b/xds/third_party/envoy/src/main/proto/envoy/config/accesslog/v3/accesslog.proto index e1b5a2e58b9..e9d815aafce 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/config/accesslog/v3/accesslog.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/config/accesslog/v3/accesslog.proto @@ -4,10 +4,12 @@ package envoy.config.accesslog.v3; import "envoy/config/core/v3/base.proto"; import "envoy/config/route/v3/route_components.proto"; +import "envoy/type/matcher/v3/metadata.proto"; import "envoy/type/v3/percent.proto"; import "google/protobuf/any.proto"; import "google/protobuf/struct.proto"; +import "google/protobuf/wrappers.proto"; import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; @@ -39,8 +41,8 @@ message AccessLog { // Filter which is used to determine if the access log needs to be written. AccessLogFilter filter = 2; - // Custom configuration that depends on the access log being instantiated. Built-in - // configurations include: + // Custom configuration that depends on the access log being instantiated. + // Built-in configurations include: // // #. "envoy.access_loggers.file": :ref:`FileAccessLog // ` @@ -53,7 +55,7 @@ message AccessLog { } } -// [#next-free-field: 12] +// [#next-free-field: 13] message AccessLogFilter { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.accesslog.v2.AccessLogFilter"; @@ -93,6 +95,9 @@ message AccessLogFilter { // Extension filter. ExtensionFilter extension_filter = 11; + + // Metadata Filter + MetadataFilter metadata_filter = 12; } } @@ -156,25 +161,30 @@ message RuntimeFilter { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.accesslog.v2.RuntimeFilter"; - // Runtime key to get an optional overridden numerator for use in the *percent_sampled* field. - // If found in runtime, this value will replace the default numerator. + // Runtime key to get an optional overridden numerator for use in the + // *percent_sampled* field. If found in runtime, this value will replace the + // default numerator. string runtime_key = 1 [(validate.rules).string = {min_bytes: 1}]; - // The default sampling percentage. If not specified, defaults to 0% with denominator of 100. + // The default sampling percentage. If not specified, defaults to 0% with + // denominator of 100. type.v3.FractionalPercent percent_sampled = 2; // By default, sampling pivots on the header - // :ref:`x-request-id` being present. If - // :ref:`x-request-id` is present, the filter will - // consistently sample across multiple hosts based on the runtime key value and the value - // extracted from :ref:`x-request-id`. If it is - // missing, or *use_independent_randomness* is set to true, the filter will randomly sample based - // on the runtime key value alone. *use_independent_randomness* can be used for logging kill - // switches within complex nested :ref:`AndFilter + // :ref:`x-request-id` being + // present. If :ref:`x-request-id` + // is present, the filter will consistently sample across multiple hosts based + // on the runtime key value and the value extracted from + // :ref:`x-request-id`. If it is + // missing, or *use_independent_randomness* is set to true, the filter will + // randomly sample based on the runtime key value alone. + // *use_independent_randomness* can be used for logging kill switches within + // complex nested :ref:`AndFilter // ` and :ref:`OrFilter - // ` blocks that are easier to reason about - // from a probability perspective (i.e., setting to true will cause the filter to behave like - // an independent random variable when composed within logical operator filters). + // ` blocks that are easier to + // reason about from a probability perspective (i.e., setting to true will + // cause the filter to behave like an independent random variable when + // composed within logical operator filters). bool use_independent_randomness = 3; } @@ -203,21 +213,22 @@ message HeaderFilter { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.accesslog.v2.HeaderFilter"; - // Only requests with a header which matches the specified HeaderMatcher will pass the filter - // check. + // Only requests with a header which matches the specified HeaderMatcher will + // pass the filter check. route.v3.HeaderMatcher header = 1 [(validate.rules).message = {required: true}]; } // Filters requests that received responses with an Envoy response flag set. // A list of the response flags can be found -// in the access log formatter :ref:`documentation`. +// in the access log formatter +// :ref:`documentation`. message ResponseFlagFilter { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.accesslog.v2.ResponseFlagFilter"; - // Only responses with the any of the flags listed in this field will be logged. - // This field is optional. If it is not specified, then any response flag will pass - // the filter check. + // Only responses with the any of the flags listed in this field will be + // logged. This field is optional. If it is not specified, then any response + // flag will pass the filter check. repeated string flags = 1 [(validate.rules).repeated = { items { string { @@ -248,8 +259,8 @@ message ResponseFlagFilter { }]; } -// Filters gRPC requests based on their response status. If a gRPC status is not provided, the -// filter will infer the status from the HTTP status code. +// Filters gRPC requests based on their response status. If a gRPC status is not +// provided, the filter will infer the status from the HTTP status code. message GrpcStatusFilter { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.accesslog.v2.GrpcStatusFilter"; @@ -277,11 +288,32 @@ message GrpcStatusFilter { // Logs only responses that have any one of the gRPC statuses in this field. repeated Status statuses = 1 [(validate.rules).repeated = {items {enum {defined_only: true}}}]; - // If included and set to true, the filter will instead block all responses with a gRPC status or - // inferred gRPC status enumerated in statuses, and allow all other responses. + // If included and set to true, the filter will instead block all responses + // with a gRPC status or inferred gRPC status enumerated in statuses, and + // allow all other responses. bool exclude = 2; } +// Filters based on matching dynamic metadata. +// If the matcher path and key correspond to an existing key in dynamic +// metadata, the request is logged only if the matcher value is equal to the +// metadata value. If the matcher path and key *do not* correspond to an +// existing key in dynamic metadata, the request is logged only if +// match_if_key_not_found is "true" or unset. +message MetadataFilter { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.filter.accesslog.v2.MetadataFilter"; + + // Matcher to check metadata for specified value. For example, to match on the + // access_log_hint metadata, set the filter to "envoy.common" and the path to + // "access_log_hint", and the value to "true". + type.matcher.v3.MetadataMatcher matcher = 1; + + // Default result if the key does not exist in dynamic metadata: if unset or + // true, then log; if false, then don't log. + google.protobuf.BoolValue match_if_key_not_found = 2; +} + // Extension filter is statically registered at runtime. message ExtensionFilter { option (udpa.annotations.versioning).previous_message_type = diff --git a/xds/third_party/envoy/src/main/proto/envoy/config/cluster/v3/cluster.proto b/xds/third_party/envoy/src/main/proto/envoy/config/cluster/v3/cluster.proto index b4ea53bb093..3571ccf9abb 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/config/cluster/v3/cluster.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/config/cluster/v3/cluster.proto @@ -43,7 +43,7 @@ message ClusterCollection { } // Configuration for a single upstream cluster. -// [#next-free-field: 50] +// [#next-free-field: 53] message Cluster { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.Cluster"; @@ -246,6 +246,20 @@ message Cluster { // List of keys to match with the weighted cluster metadata. repeated string keys = 1; + // Selects a mode of operation in which each subset has only one host. This mode uses the same rules for + // choosing a host, but updating hosts is faster, especially for large numbers of hosts. + // + // If a match is found to a host, that host will be used regardless of priority levels, unless the host is unhealthy. + // + // Currently, this mode is only supported if `subset_selectors` has only one entry, and `keys` contains + // only one entry. + // + // When this mode is enabled, configurations that contain more than one host with the same metadata value for the single key in `keys` + // will use only one of the hosts with the given key; no requests will be routed to the others. The cluster gauge + // :ref:`lb_subsets_single_host_per_subset_duplicate` indicates how many duplicates are + // present in the current configuration. + bool single_host_per_subset = 4; + // The behavior used when no endpoint subset matches the selected route's // metadata. LbSubsetSelectorFallbackPolicy fallback_policy = 2 @@ -396,6 +410,16 @@ message Cluster { google.protobuf.UInt64Value maximum_ring_size = 4 [(validate.rules).uint64 = {lte: 8388608}]; } + // Specific configuration for the :ref:`Maglev` + // load balancing policy. + message MaglevLbConfig { + // The table size for Maglev hashing. The Maglev aims for ‘minimal disruption’ rather than an absolute guarantee. + // Minimal disruption means that when the set of upstreams changes, a connection will likely be sent to the same + // upstream as it was before. Increasing the table size reduces the amount of disruption. + // The table size must be prime number. If it is not specified, the default is 65537. + google.protobuf.UInt64Value table_size = 1; + } + // Specific configuration for the // :ref:`Original Destination ` // load balancing policy. @@ -462,6 +486,26 @@ message Cluster { // If set to `true`, the cluster will use hostname instead of the resolved // address as the key to consistently hash to an upstream host. Only valid for StrictDNS clusters with hostnames which resolve to a single IP address. bool use_hostname_for_hashing = 1; + + // Configures percentage of average cluster load to bound per upstream host. For example, with a value of 150 + // no upstream host will get a load more than 1.5 times the average load of all the hosts in the cluster. + // If not specified, the load is not bounded for any upstream host. Typical value for this parameter is between 120 and 200. + // Minimum is 100. + // + // Applies to both Ring Hash and Maglev load balancers. + // + // This is implemented based on the method described in the paper https://arxiv.org/abs/1608.01350. For the specified + // `hash_balance_factor`, requests to any upstream host are capped at `hash_balance_factor/100` times the average number of requests + // across the cluster. When a request arrives for an upstream host that is currently serving at its max capacity, linear probing + // is used to identify an eligible host. Further, the linear probe is implemented using a random jump in hosts ring/table to identify + // the eligible host (this technique is as described in the paper https://arxiv.org/abs/1908.08762 - the random jump avoids the + // cascading overflow effect when choosing the next host in the ring/table). + // + // If weights are specified on the hosts, they are respected. + // + // This is an O(N) algorithm, unlike other load balancers. Using a lower `hash_balance_factor` results in more hosts + // being probed, so use a higher value if you require better performance. + google.protobuf.UInt32Value hash_balance_factor = 2 [(validate.rules).uint32 = {gte: 100}]; } // Configures the :ref:`healthy panic threshold `. @@ -519,7 +563,7 @@ message Cluster { // connections to upstream hosts whenever hosts are added or removed from the cluster. bool close_connections_on_host_set_change = 6; - //Common Configuration for all consistent hashing load balancers (MaglevLb, RingHashLb, etc.) + // Common Configuration for all consistent hashing load balancers (MaglevLb, RingHashLb, etc.) ConsistentHashingLbConfig consistent_hashing_lb_config = 7; } @@ -541,6 +585,36 @@ message Cluster { google.protobuf.Duration max_interval = 2 [(validate.rules).duration = {gt {nanos: 1000000}}]; } + // [#not-implemented-hide:] + message PrefetchPolicy { + // Indicates how many streams (rounded up) can be anticipated per-upstream for each + // incoming stream. This is useful for high-QPS or latency-sensitive services. Prefetching + // will only be done if the upstream is healthy. + // + // For example if this is 2, for an incoming HTTP/1.1 stream, 2 connections will be + // established, one for the new incoming stream, and one for a presumed follow-up stream. For + // HTTP/2, only one connection would be established by default as one connection can + // serve both the original and presumed follow-up stream. + // + // In steady state for non-multiplexed connections a value of 1.5 would mean if there were 100 + // active streams, there would be 100 connections in use, and 50 connections prefetched. + // This might be a useful value for something like short lived single-use connections, + // for example proxying HTTP/1.1 if keep-alive were false and each stream resulted in connection + // termination. It would likely be overkill for long lived connections, such as TCP proxying SMTP + // or regular HTTP/1.1 with keep-alive. For long lived traffic, a value of 1.05 would be more + // reasonable, where for every 100 connections, 5 prefetched connections would be in the queue + // in case of unexpected disconnects where the connection could not be reused. + // + // If this value is not set, or set explicitly to one, Envoy will fetch as many connections + // as needed to serve streams in flight. This means in steady state if a connection is torn down, + // a subsequent streams will pay an upstream-rtt latency penalty waiting for streams to be + // prefetched. + // + // This is limited somewhat arbitrarily to 3 because prefetching connections too aggressively can + // harm latency more than the prefetching helps. + google.protobuf.DoubleValue prefetch_ratio = 1 [(validate.rules).double = {lte: 3.0 gte: 1.0}]; + } + reserved 12, 15, 7, 11, 35; reserved "hosts", "tls_context", "extension_protocol_options"; @@ -766,15 +840,19 @@ message Cluster { // Optional configuration for the load balancing algorithm selected by // LbPolicy. Currently only - // :ref:`RING_HASH` and + // :ref:`RING_HASH`, + // :ref:`MAGLEV` and // :ref:`LEAST_REQUEST` // has additional configuration options. - // Specifying ring_hash_lb_config or least_request_lb_config without setting the corresponding + // Specifying ring_hash_lb_config or maglev_lb_config or least_request_lb_config without setting the corresponding // LbPolicy will generate an error at runtime. oneof lb_config { // Optional configuration for the Ring Hash load balancing policy. RingHashLbConfig ring_hash_lb_config = 23; + // Optional configuration for the Maglev load balancing policy. + MaglevLbConfig maglev_lb_config = 52; + // Optional configuration for the Original Destination load balancing policy. OriginalDstLbConfig original_dst_lb_config = 34; @@ -884,6 +962,14 @@ message Cluster { // Configuration to track optional cluster stats. TrackClusterStats track_cluster_stats = 49; + + // [#not-implemented-hide:] + // Prefetch configuration for this cluster. + PrefetchPolicy prefetch_policy = 50; + + // If `connection_pool_per_downstream_connection` is true, the cluster will use a separate + // connection pool for every downstream connection + bool connection_pool_per_downstream_connection = 51; } // [#not-implemented-hide:] Extensible load balancing policy configuration. diff --git a/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/address.proto b/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/address.proto index 5102c2d5759..030b68e6948 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/address.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/address.proto @@ -30,6 +30,17 @@ message Pipe { uint32 mode = 2 [(validate.rules).uint32 = {lte: 511}]; } +// [#not-implemented-hide:] The address represents an envoy internal listener. +// TODO(lambdai): Make this address available for listener and endpoint. +message EnvoyInternalAddress { + oneof address_name_specifier { + option (validate.required) = true; + + // [#not-implemented-hide:] The :ref:`listener name ` of the destination internal listener. + string server_listener_name = 1; + } +} + // [#next-free-field: 7] message SocketAddress { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.core.SocketAddress"; @@ -129,6 +140,9 @@ message Address { SocketAddress socket_address = 1; Pipe pipe = 2; + + // [#not-implemented-hide:] + EnvoyInternalAddress envoy_internal_address = 3; } } diff --git a/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/base.proto b/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/base.proto index 4509c166256..15a17b49384 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/base.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/base.proto @@ -196,7 +196,7 @@ message Node { // for filtering :ref:`listeners ` to be returned. For example, // if there is a listener bound to port 80, the list can optionally contain the // SocketAddress `(0.0.0.0,80)`. The field is optional and just a hint. - repeated Address listening_addresses = 11; + repeated Address listening_addresses = 11 [deprecated = true]; } // Metadata provides additional inputs to filters based on matched listeners, @@ -293,7 +293,7 @@ message HeaderValueOption { HeaderValue header = 1 [(validate.rules).message = {required: true}]; // Should the value be appended? If true (default), the value is appended to - // existing values. + // existing values. Otherwise it replaces any existing values. google.protobuf.BoolValue append = 2; } diff --git a/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/grpc_service.proto b/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/grpc_service.proto index 3f62884df6e..967c694d2bc 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/grpc_service.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/grpc_service.proto @@ -36,6 +36,12 @@ message GrpcService { // in the :ref:`Cluster ` :ref:`transport_socket // `. string cluster_name = 1 [(validate.rules).string = {min_bytes: 1}]; + + // The `:authority` header in the grpc request. If this field is not set, the authority header value will be `cluster_name`. + // Note that this authority does not override the SNI. The SNI is provided by the transport socket of the cluster. + string authority = 2 + [(validate.rules).string = + {min_bytes: 0 max_bytes: 16384 well_known_regex: HTTP_HEADER_VALUE strict: false}]; } // [#next-free-field: 9] diff --git a/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/protocol.proto b/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/protocol.proto index 0ab6289e965..3e20f3b449a 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/protocol.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/protocol.proto @@ -100,7 +100,7 @@ message HttpProtocolOptions { HeadersWithUnderscoresAction headers_with_underscores_action = 5; } -// [#next-free-field: 6] +// [#next-free-field: 8] message Http1ProtocolOptions { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.core.Http1ProtocolOptions"; @@ -157,6 +157,24 @@ message Http1ProtocolOptions { // - Not a response to a HEAD request. // - The content length header is not present. bool enable_trailers = 5; + + // Allows Envoy to process requests/responses with both `Content-Length` and `Transfer-Encoding` + // headers set. By default such messages are rejected, but if option is enabled - Envoy will + // remove Content-Length header and process message. + // See `RFC7230, sec. 3.3.3 ` for details. + // + // .. attention:: + // Enabling this option might lead to request smuggling vulnerability, especially if traffic + // is proxied via multiple layers of proxies. + bool allow_chunked_length = 6; + + // Allows invalid HTTP messaging. When this option is false, then Envoy will terminate + // HTTP/1.1 connections upon receiving an invalid HTTP message. However, + // when this option is true, then Envoy will leave the HTTP/1.1 connection + // open where possible. + // If set, this overrides any HCM :ref:`stream_error_on_invalid_http_messaging + // `. + google.protobuf.BoolValue override_stream_error_on_invalid_http_message = 7; } // [#next-free-field: 15] diff --git a/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/substitution_format_string.proto b/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/substitution_format_string.proto index 7537a1178b6..6c129707b2e 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/substitution_format_string.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/substitution_format_string.proto @@ -58,4 +58,11 @@ message SubstitutionFormatString { // google.protobuf.Struct json_format = 2 [(validate.rules).message = {required: true}]; } + + // If set to true, when command operators are evaluated to null, + // + // * for ``text_format``, the output of the empty operator is changed from ``-`` to an + // empty string, so that empty values are omitted entirely. + // * for ``json_format`` the keys with null values are omitted in the output structure. + bool omit_empty_values = 3; } diff --git a/xds/third_party/envoy/src/main/proto/envoy/config/listener/v3/listener.proto b/xds/third_party/envoy/src/main/proto/envoy/config/listener/v3/listener.proto index ab0b0ecac7c..dab0eb1ce68 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/config/listener/v3/listener.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/config/listener/v3/listener.proto @@ -5,6 +5,7 @@ package envoy.config.listener.v3; import "envoy/config/accesslog/v3/accesslog.proto"; import "envoy/config/core/v3/address.proto"; import "envoy/config/core/v3/base.proto"; +import "envoy/config/core/v3/extension.proto"; import "envoy/config/core/v3/socket_option.proto"; import "envoy/config/listener/v3/api_listener.proto"; import "envoy/config/listener/v3/listener_components.proto"; @@ -32,10 +33,10 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // Listener list collections. Entries are *Listener* resources or references. // [#not-implemented-hide:] message ListenerCollection { - udpa.core.v1.CollectionEntry entries = 1; + repeated udpa.core.v1.CollectionEntry entries = 1; } -// [#next-free-field: 23] +// [#next-free-field: 25] message Listener { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.Listener"; @@ -248,4 +249,18 @@ message Listener { // Configuration for :ref:`access logs ` // emitted by this listener. repeated accesslog.v3.AccessLog access_log = 22; + + // If the protocol in the listener socket address in :ref:`protocol + // ` is :ref:`UDP + // `, this field specifies the actual udp + // writer to create, i.e. :ref:`name ` + // = "udp_default_writer" for creating a udp writer with writing in passthrough mode, + // = "udp_gso_batch_writer" for creating a udp writer with writing in batch mode. + // If not present, treat it as "udp_default_writer". + // [#not-implemented-hide:] + core.v3.TypedExtensionConfig udp_writer_config = 23; + + // The maximum length a tcp listener's pending connections queue can grow to. If no value is + // provided net.core.somaxconn will be used on Linux and 128 otherwise. + google.protobuf.UInt32Value tcp_backlog_size = 24; } diff --git a/xds/third_party/envoy/src/main/proto/envoy/config/listener/v3/listener_components.proto b/xds/third_party/envoy/src/main/proto/envoy/config/listener/v3/listener_components.proto index 88e75e65b52..8a22fbc97f5 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/config/listener/v3/listener_components.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/config/listener/v3/listener_components.proto @@ -7,6 +7,7 @@ import "envoy/config/core/v3/base.proto"; import "envoy/type/v3/range.proto"; import "google/protobuf/any.proto"; +import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; @@ -167,10 +168,25 @@ message FilterChainMatch { // A filter chain wraps a set of match criteria, an option TLS context, a set of filters, and // various other parameters. -// [#next-free-field: 8] +// [#next-free-field: 9] message FilterChain { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.listener.FilterChain"; + // The configuration for on-demand filter chain. If this field is not empty in FilterChain message, + // a filter chain will be built on-demand. + // On-demand filter chains help speedup the warming up of listeners since the building and initialization of + // an on-demand filter chain will be postponed to the arrival of new connection requests that require this filter chain. + // Filter chains that are not often used can be set as on-demand. + message OnDemandConfiguration { + // The timeout to wait for filter chain placeholders to complete rebuilding. + // 1. If this field is set to 0, timeout is disabled. + // 2. If not specified, a default timeout of 15s is used. + // Rebuilding will wait until dependencies are ready, have failed, or this timeout is reached. + // Upon failure or timeout, all connections related to this filter chain will be closed. + // Rebuilding will start again on the next new connection. + google.protobuf.Duration rebuild_timeout = 1; + } + reserved 2; reserved "tls_context"; @@ -206,6 +222,11 @@ message FilterChain { // name is provided, Envoy will allocate an internal UUID for the filter chain. If the filter // chain is to be dynamically updated or removed via FCDS a unique name must be provided. string name = 7; + + // [#not-implemented-hide:] The configuration to specify whether the filter chain will be built on-demand. + // If this field is not empty, the filter chain will be built on-demand. + // Otherwise, the filter chain will be built normally and block listener warming. + OnDemandConfiguration on_demand_configuration = 8; } // Listener filter chain match configuration. This is a recursive structure which allows complex diff --git a/xds/third_party/envoy/src/main/proto/envoy/config/route/v3/route_components.proto b/xds/third_party/envoy/src/main/proto/envoy/config/route/v3/route_components.proto index c35e210691c..0d1d7cf4c43 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/config/route/v3/route_components.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/config/route/v3/route_components.proto @@ -338,7 +338,9 @@ message WeightedCluster { // Specifies a list of HTTP headers that should be removed from each request when // this cluster is selected through the enclosing :ref:`envoy_api_msg_config.route.v3.RouteAction`. - repeated string request_headers_to_remove = 9; + repeated string request_headers_to_remove = 9 [(validate.rules).repeated = { + items {string {well_known_regex: HTTP_HEADER_NAME strict: false}} + }]; // Specifies a list of headers to be added to responses when this cluster is selected // through the enclosing :ref:`envoy_api_msg_config.route.v3.RouteAction`. @@ -352,7 +354,9 @@ message WeightedCluster { // Specifies a list of headers to be removed from responses when this cluster is selected // through the enclosing :ref:`envoy_api_msg_config.route.v3.RouteAction`. - repeated string response_headers_to_remove = 6; + repeated string response_headers_to_remove = 6 [(validate.rules).repeated = { + items {string {well_known_regex: HTTP_HEADER_NAME strict: false}} + }]; // The per_filter_config field can be used to provide weighted cluster-specific // configurations for filters. The key should match the filter name, such as @@ -440,14 +444,14 @@ message RouteMatch { // (WebSocket and the like) as they are normalized in Envoy as HTTP/1.1 style // upgrades. // This is the only way to match CONNECT requests for HTTP/1.1. For HTTP/2, - // where CONNECT requests may have a path, the path matchers will work if + // where Extended CONNECT requests may have a path, the path matchers will work if // there is a path present. // Note that CONNECT support is currently considered alpha in Envoy. // [#comment:TODO(htuch): Replace the above comment with an alpha tag. ConnectMatcher connect_matcher = 12; } - // Indicates that prefix/path matching should be case insensitive. The default + // Indicates that prefix/path matching should be case sensitive. The default // is true. google.protobuf.BoolValue case_sensitive = 4; @@ -545,7 +549,7 @@ message CorsPolicy { core.v3.RuntimeFractionalPercent shadow_enabled = 10; } -// [#next-free-field: 35] +// [#next-free-field: 37] message RouteAction { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.route.RouteAction"; @@ -760,6 +764,31 @@ message RouteAction { ConnectConfig connect_config = 3; } + // [#not-implemented-hide:] + message MaxStreamDuration { + // Specifies the maximum duration allowed for streams on the route. If not specified, the value + // from the :ref:`max_stream_duration + // ` field in + // :ref:`HttpConnectionManager.common_http_protocol_options + // ` + // is used. + google.protobuf.Duration max_stream_duration = 1; + + // If present, and the request contains a `grpc-timeout header + // `_, use that value as the + // *max_stream_duration*, but limit the applied timeout to the maximum value specified here. + // If set to 0, the `grpc-timeout` header is used without modification. + google.protobuf.Duration grpc_max_timeout = 2; + + // If present, Envoy will adjust the timeout provided by the `grpc-timeout` header by + // subtracting the provided duration from the header. This is useful for allowing Envoy to set + // its global timeout to be less than that of the deadline imposed by the calling client, which + // makes it more likely that Envoy will handle the timeout instead of having the call canceled + // by the client. If, after applying the offset, the resulting timeout is zero or negative, + // the stream will timeout immediately. + google.protobuf.Duration grpc_timeout_offset = 3; + } + reserved 12, 18, 19, 16, 22, 21, 10; reserved "request_mirror_policy"; @@ -890,6 +919,23 @@ message RouteAction { // must come from trusted source. string host_rewrite_header = 29 [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME strict: false}]; + + // Indicates that during forwarding, the host header will be swapped with + // the result of the regex substitution executed on path value with query and fragment removed. + // This is useful for transitioning variable content between path segment and subdomain. + // + // For example with the following config: + // + // .. code-block:: yaml + // + // host_rewrite_path_regex: + // pattern: + // google_re2: {} + // regex: "^/(.+)/.+$" + // substitution: \1 + // + // Would rewrite the host header to `envoyproxy.io` given the path `/envoyproxy.io/some/path`. + type.matcher.v3.RegexMatchAndSubstitute host_rewrite_path_regex = 35; } // Specifies the upstream timeout for the route. If not specified, the default is 15s. This @@ -951,7 +997,9 @@ message RouteAction { // limits. By default, if the route configured rate limits, the virtual host // :ref:`rate_limits ` are not applied to the // request. - google.protobuf.BoolValue include_vh_rate_limits = 14; + // + // This field is deprecated. Please use :ref:`vh_rate_limits ` + google.protobuf.BoolValue include_vh_rate_limits = 14 [deprecated = true]; // Specifies a list of hash policies to use for ring hash load balancing. Each // hash policy is evaluated individually and the combined result is used to @@ -1030,13 +1078,22 @@ message RouteAction { // it'll take precedence over the virtual host level hedge policy entirely // (e.g.: policies are not merged, most internal one becomes the enforced policy). HedgePolicy hedge_policy = 27; + + // Specifies the maximum stream duration for this route. + // [#not-implemented-hide:] + MaxStreamDuration max_stream_duration = 36; } // HTTP retry :ref:`architecture overview `. -// [#next-free-field: 11] +// [#next-free-field: 12] message RetryPolicy { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.route.RetryPolicy"; + enum ResetHeaderFormat { + SECONDS = 0; + UNIX_TIMESTAMP = 1; + } + message RetryPriority { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.route.RetryPolicy.RetryPriority"; @@ -1087,6 +1144,69 @@ message RetryPolicy { google.protobuf.Duration max_interval = 2 [(validate.rules).duration = {gt {}}]; } + message ResetHeader { + string name = 1 + [(validate.rules).string = {min_bytes: 1 well_known_regex: HTTP_HEADER_NAME strict: false}]; + + ResetHeaderFormat format = 2 [(validate.rules).enum = {defined_only: true}]; + } + + // A retry back-off strategy that applies when the upstream server rate limits + // the request. + // + // Given this configuration: + // + // .. code-block:: yaml + // + // rate_limited_retry_back_off: + // reset_headers: + // - name: Retry-After + // format: SECONDS + // - name: X-RateLimit-Reset + // format: UNIX_TIMESTAMP + // max_interval: "300s" + // + // The following algorithm will apply: + // + // 1. If the response contains the header ``Retry-After`` its value must be on + // the form ``120`` (an integer that represents the number of seconds to + // wait before retrying). If so, this value is used as the back-off interval. + // 2. Otherwise, if the response contains the header ``X-RateLimit-Reset`` its + // value must be on the form ``1595320702`` (an integer that represents the + // point in time at which to retry, as a Unix timestamp in seconds). If so, + // the current time is subtracted from this value and the result is used as + // the back-off interval. + // 3. Otherwise, Envoy will use the default + // :ref:`exponential back-off ` + // strategy. + // + // No matter which format is used, if the resulting back-off interval exceeds + // ``max_interval`` it is discarded and the next header in ``reset_headers`` + // is tried. If a request timeout is configured for the route it will further + // limit how long the request will be allowed to run. + // + // To prevent many clients retrying at the same point in time jitter is added + // to the back-off interval, so the resulting interval is decided by taking: + // ``random(interval, interval * 1.5)``. + // + // .. attention:: + // + // Configuring ``rate_limited_retry_back_off`` will not by itself cause a request + // to be retried. You will still need to configure the right retry policy to match + // the responses from the upstream server. + message RateLimitedRetryBackOff { + // Specifies the reset headers (like ``Retry-After`` or ``X-RateLimit-Reset``) + // to match against the response. Headers are tried in order, and matched case + // insensitive. The first header to be parsed successfully is used. If no headers + // match the default exponential back-off is used instead. + repeated ResetHeader reset_headers = 1 [(validate.rules).repeated = {min_items: 1}]; + + // Specifies the maximum back off interval that Envoy will allow. If a reset + // header contains an interval longer than this then it will be discarded and + // the next header will be tried. Defaults to 300 seconds. + google.protobuf.Duration max_interval = 2 [(validate.rules).duration = {gt {}}]; + } + // Specifies the conditions under which retry takes place. These are the same // conditions documented for :ref:`config_http_filters_router_x-envoy-retry-on` and // :ref:`config_http_filters_router_x-envoy-retry-grpc-on`. @@ -1130,13 +1250,22 @@ message RetryPolicy { // HTTP status codes that should trigger a retry in addition to those specified by retry_on. repeated uint32 retriable_status_codes = 7; - // Specifies parameters that control retry back off. This parameter is optional, in which case the + // Specifies parameters that control exponential retry back off. This parameter is optional, in which case the // default base interval is 25 milliseconds or, if set, the current value of the // `upstream.base_retry_backoff_ms` runtime parameter. The default maximum interval is 10 times // the base interval. The documentation for :ref:`config_http_filters_router_x-envoy-max-retries` // describes Envoy's back-off algorithm. RetryBackOff retry_back_off = 8; + // Specifies parameters that control a retry back-off strategy that is used + // when the request is rate limited by the upstream server. The server may + // return a response header like ``Retry-After`` or ``X-RateLimit-Reset`` to + // provide feedback to the client on how long to wait before retrying. If + // configured, this back-off strategy will be used instead of the + // default exponential back off strategy (configured using `retry_back_off`) + // whenever a response includes the matching headers. + RateLimitedRetryBackOff rate_limited_retry_back_off = 11; + // HTTP response headers that trigger a retry if present in the response. A retry will be // triggered if any of the header matches match the upstream response headers. // The field is only consulted if 'retriable-headers' retry policy is active. @@ -1452,6 +1581,10 @@ message RateLimit { // The value to use in the descriptor entry. string descriptor_value = 1 [(validate.rules).string = {min_bytes: 1}]; + + // An optional key to use in the descriptor entry. If not set it defaults + // to 'generic_key' as the descriptor key. + string descriptor_key = 2; } // The following descriptor entry is appended to the descriptor: @@ -1593,7 +1726,7 @@ message RateLimit { // value. // // [#next-major-version: HeaderMatcher should be refactored to use StringMatcher.] -// [#next-free-field: 12] +// [#next-free-field: 13] message HeaderMatcher { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.route.HeaderMatcher"; @@ -1647,6 +1780,15 @@ message HeaderMatcher { // // * The suffix *abcd* matches the value *xyzabcd*, but not for *xyzbcd*. string suffix_match = 10 [(validate.rules).string = {min_bytes: 1}]; + + // If specified, header match will be performed based on whether the header value contains + // the given value or not. + // Note: empty contains match is not allowed, please use present_match instead. + // + // Examples: + // + // * The value *abcd* matches the value *xyzabcdpqr*, but not for *xyzbcdpqr*. + string contains_match = 12 [(validate.rules).string = {min_bytes: 1}]; } // If specified, the match result will be inverted before checking. Defaults to false. diff --git a/xds/third_party/envoy/src/main/proto/envoy/config/route/v3/scoped_route.proto b/xds/third_party/envoy/src/main/proto/envoy/config/route/v3/scoped_route.proto index f2b28ed974c..d6611b0b1d0 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/config/route/v3/scoped_route.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/config/route/v3/scoped_route.proto @@ -104,6 +104,9 @@ message ScopedRouteConfiguration { repeated Fragment fragments = 1 [(validate.rules).repeated = {min_items: 1}]; } + // Whether the RouteConfiguration should be loaded on demand. + bool on_demand = 4; + // The name assigned to the routing scope. string name = 1 [(validate.rules).string = {min_bytes: 1}]; diff --git a/xds/third_party/envoy/src/main/proto/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto b/xds/third_party/envoy/src/main/proto/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto index 04a132ad267..68c5c8cad2a 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto @@ -298,7 +298,8 @@ message HttpConnectionManager { // An optional override that the connection manager will write to the server // header in responses. If not set, the default is *envoy*. - string server_name = 10; + string server_name = 10 + [(validate.rules).string = {well_known_regex: HTTP_HEADER_VALUE strict: false}]; // Defines the action to be applied to the Server header on the response path. // By default, Envoy will overwrite the header with the value specified in @@ -552,7 +553,8 @@ message HttpConnectionManager { // company-internal mesh) and false when receiving untrusted traffic (edge deployments). // // If different behaviors for invalid_http_message for HTTP/1 and HTTP/2 are - // desired, one *must* use the new HTTP/2 option + // desired, one should use the new HTTP/1 option :ref:`override_stream_error_on_invalid_http_message + // ` or the new HTTP/2 option // :ref:`override_stream_error_on_invalid_http_message // ` // *not* the deprecated but similarly named :ref:`stream_error_on_invalid_http_messaging diff --git a/xds/third_party/envoy/src/main/proto/envoy/extensions/transport_sockets/tls/v3/common.proto b/xds/third_party/envoy/src/main/proto/envoy/extensions/transport_sockets/tls/v3/common.proto index 115ecad72f9..5eab3c1060b 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/extensions/transport_sockets/tls/v3/common.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/extensions/transport_sockets/tls/v3/common.proto @@ -224,7 +224,10 @@ message CertificateValidationContext { // specified. // // It can optionally contain certificate revocation lists, in which case Envoy will verify - // that the presented peer certificate has not been revoked by one of the included CRLs. + // that the presented peer certificate has not been revoked by one of the included CRLs. Note + // that if a CRL is provided for any certificate authority in a trust chain, a CRL must be + // provided for all certificate authorities in that chain. Failure to do so will result in + // verification failure for both revoked and unrevoked certificates from that chain. // // See :ref:`the TLS overview ` for a list of common // system CA locations. @@ -322,7 +325,11 @@ message CertificateValidationContext { // `_ // (in PEM format). If specified, Envoy will verify that the presented peer // certificate has not been revoked by this CRL. If this DataSource contains - // multiple CRLs, all of them will be used. + // multiple CRLs, all of them will be used. Note that if a CRL is provided + // for any certificate authority in a trust chain, a CRL must be provided + // for all certificate authorities in that chain. Failure to do so will + // result in verification failure for both revoked and unrevoked certificates + // from that chain. config.core.v3.DataSource crl = 7; // If specified, Envoy will not reject expired certificates. diff --git a/xds/third_party/envoy/src/main/proto/envoy/extensions/transport_sockets/tls/v3/tls.proto b/xds/third_party/envoy/src/main/proto/envoy/extensions/transport_sockets/tls/v3/tls.proto index 7ee7920c724..f746f3d2f1c 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/extensions/transport_sockets/tls/v3/tls.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/extensions/transport_sockets/tls/v3/tls.proto @@ -99,7 +99,7 @@ message DownstreamTlsContext { } // TLS context shared by both client and server TLS contexts. -// [#next-free-field: 13] +// [#next-free-field: 14] message CommonTlsContext { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.auth.CommonTlsContext"; @@ -238,4 +238,8 @@ message CommonTlsContext { // // There is no default for this parameter. If empty, Envoy will not expose ALPN. repeated string alpn_protocols = 4; + + // Custom TLS handshaker. If empty, defaults to native TLS handshaking + // behavior. + config.core.v3.TypedExtensionConfig custom_handshaker = 13; } diff --git a/xds/third_party/envoy/src/main/proto/envoy/type/matcher/v3/metadata.proto b/xds/third_party/envoy/src/main/proto/envoy/type/matcher/v3/metadata.proto new file mode 100644 index 00000000000..65ec4f47fff --- /dev/null +++ b/xds/third_party/envoy/src/main/proto/envoy/type/matcher/v3/metadata.proto @@ -0,0 +1,104 @@ +syntax = "proto3"; + +package envoy.type.matcher.v3; + +import "envoy/type/matcher/v3/value.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.type.matcher.v3"; +option java_outer_classname = "MetadataProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Metadata matcher] + +// MetadataMatcher provides a general interface to check if a given value is matched in +// :ref:`Metadata `. It uses `filter` and `path` to retrieve the value +// from the Metadata and then check if it's matched to the specified value. +// +// For example, for the following Metadata: +// +// .. code-block:: yaml +// +// filter_metadata: +// envoy.filters.http.rbac: +// fields: +// a: +// struct_value: +// fields: +// b: +// struct_value: +// fields: +// c: +// string_value: pro +// t: +// list_value: +// values: +// - string_value: m +// - string_value: n +// +// The following MetadataMatcher is matched as the path [a, b, c] will retrieve a string value "pro" +// from the Metadata which is matched to the specified prefix match. +// +// .. code-block:: yaml +// +// filter: envoy.filters.http.rbac +// path: +// - key: a +// - key: b +// - key: c +// value: +// string_match: +// prefix: pr +// +// The following MetadataMatcher is matched as the code will match one of the string values in the +// list at the path [a, t]. +// +// .. code-block:: yaml +// +// filter: envoy.filters.http.rbac +// path: +// - key: a +// - key: t +// value: +// list_match: +// one_of: +// string_match: +// exact: m +// +// An example use of MetadataMatcher is specifying additional metadata in envoy.filters.http.rbac to +// enforce access control based on dynamic metadata in a request. See :ref:`Permission +// ` and :ref:`Principal +// `. + +// [#next-major-version: MetadataMatcher should use StructMatcher] +message MetadataMatcher { + option (udpa.annotations.versioning).previous_message_type = "envoy.type.matcher.MetadataMatcher"; + + // Specifies the segment in a path to retrieve value from Metadata. + // Note: Currently it's not supported to retrieve a value from a list in Metadata. This means that + // if the segment key refers to a list, it has to be the last segment in a path. + message PathSegment { + option (udpa.annotations.versioning).previous_message_type = + "envoy.type.matcher.MetadataMatcher.PathSegment"; + + oneof segment { + option (validate.required) = true; + + // If specified, use the key to retrieve the value in a Struct. + string key = 1 [(validate.rules).string = {min_bytes: 1}]; + } + } + + // The filter name to retrieve the Struct from the Metadata. + string filter = 1 [(validate.rules).string = {min_bytes: 1}]; + + // The path to retrieve the Value from the Struct. + repeated PathSegment path = 2 [(validate.rules).repeated = {min_items: 1}]; + + // The MetadataMatcher is matched if the value retrieved by path is matched to this value. + ValueMatcher value = 3 [(validate.rules).message = {required: true}]; +} diff --git a/xds/third_party/envoy/src/main/proto/envoy/type/matcher/v3/number.proto b/xds/third_party/envoy/src/main/proto/envoy/type/matcher/v3/number.proto new file mode 100644 index 00000000000..2379efdcbd2 --- /dev/null +++ b/xds/third_party/envoy/src/main/proto/envoy/type/matcher/v3/number.proto @@ -0,0 +1,32 @@ +syntax = "proto3"; + +package envoy.type.matcher.v3; + +import "envoy/type/v3/range.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.type.matcher.v3"; +option java_outer_classname = "NumberProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Number matcher] + +// Specifies the way to match a double value. +message DoubleMatcher { + option (udpa.annotations.versioning).previous_message_type = "envoy.type.matcher.DoubleMatcher"; + + oneof match_pattern { + option (validate.required) = true; + + // If specified, the input double value must be in the range specified here. + // Note: The range is using half-open interval semantics [start, end). + type.v3.DoubleRange range = 1; + + // If specified, the input double value must be equal to the value specified here. + double exact = 2; + } +} diff --git a/xds/third_party/envoy/src/main/proto/envoy/type/matcher/v3/string.proto b/xds/third_party/envoy/src/main/proto/envoy/type/matcher/v3/string.proto index 77fe48ac74c..d453d43d3f8 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/type/matcher/v3/string.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/type/matcher/v3/string.proto @@ -17,7 +17,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: String matcher] // Specifies the way to match a string. -// [#next-free-field: 7] +// [#next-free-field: 8] message StringMatcher { option (udpa.annotations.versioning).previous_message_type = "envoy.type.matcher.StringMatcher"; @@ -53,6 +53,14 @@ message StringMatcher { // The input string must match the regular expression specified here. RegexMatcher safe_regex = 5 [(validate.rules).message = {required: true}]; + + // The input string must have the substring specified here. + // Note: empty contains match is not allowed, please use regex instead. + // + // Examples: + // + // * *abc* matches the value *xyz.abc.def* + string contains = 7 [(validate.rules).string = {min_bytes: 1}]; } // If true, indicates the exact/prefix/suffix matching should be case insensitive. This has no diff --git a/xds/third_party/envoy/src/main/proto/envoy/type/matcher/v3/value.proto b/xds/third_party/envoy/src/main/proto/envoy/type/matcher/v3/value.proto new file mode 100644 index 00000000000..040332273ba --- /dev/null +++ b/xds/third_party/envoy/src/main/proto/envoy/type/matcher/v3/value.proto @@ -0,0 +1,71 @@ +syntax = "proto3"; + +package envoy.type.matcher.v3; + +import "envoy/type/matcher/v3/number.proto"; +import "envoy/type/matcher/v3/string.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.type.matcher.v3"; +option java_outer_classname = "ValueProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Value matcher] + +// Specifies the way to match a ProtobufWkt::Value. Primitive values and ListValue are supported. +// StructValue is not supported and is always not matched. +// [#next-free-field: 7] +message ValueMatcher { + option (udpa.annotations.versioning).previous_message_type = "envoy.type.matcher.ValueMatcher"; + + // NullMatch is an empty message to specify a null value. + message NullMatch { + option (udpa.annotations.versioning).previous_message_type = + "envoy.type.matcher.ValueMatcher.NullMatch"; + } + + // Specifies how to match a value. + oneof match_pattern { + option (validate.required) = true; + + // If specified, a match occurs if and only if the target value is a NullValue. + NullMatch null_match = 1; + + // If specified, a match occurs if and only if the target value is a double value and is + // matched to this field. + DoubleMatcher double_match = 2; + + // If specified, a match occurs if and only if the target value is a string value and is + // matched to this field. + StringMatcher string_match = 3; + + // If specified, a match occurs if and only if the target value is a bool value and is equal + // to this field. + bool bool_match = 4; + + // If specified, value match will be performed based on whether the path is referring to a + // valid primitive value in the metadata. If the path is referring to a non-primitive value, + // the result is always not matched. + bool present_match = 5; + + // If specified, a match occurs if and only if the target value is a list value and + // is matched to this field. + ListMatcher list_match = 6; + } +} + +// Specifies the way to match a list value. +message ListMatcher { + option (udpa.annotations.versioning).previous_message_type = "envoy.type.matcher.ListMatcher"; + + oneof match_pattern { + option (validate.required) = true; + + // If specified, at least one of the values in the list must match the value specified. + ValueMatcher one_of = 1; + } +} From ef7846496c71c6baa2a1f42ef73ca0b8ad26066e Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Tue, 22 Sep 2020 10:48:31 -0400 Subject: [PATCH 58/86] core: Inline AbstractManagedChannelImplBuilder --- .../AbstractManagedChannelImplBuilder.java | 670 ------------------ .../io/grpc/internal/ManagedChannelImpl.java | 46 +- .../internal/ManagedChannelImplBuilder.java | 617 +++++++++++++++- ...AbstractManagedChannelImplBuilderTest.java | 482 ------------- .../ManagedChannelImplBuilderTest.java | 512 ++++++++++++- .../ManagedChannelImplIdlenessTest.java | 9 +- .../grpc/internal/ManagedChannelImplTest.java | 10 +- .../ServiceConfigErrorHandlingTest.java | 10 +- 8 files changed, 1102 insertions(+), 1254 deletions(-) delete mode 100644 core/src/main/java/io/grpc/internal/AbstractManagedChannelImplBuilder.java delete mode 100644 core/src/test/java/io/grpc/internal/AbstractManagedChannelImplBuilderTest.java diff --git a/core/src/main/java/io/grpc/internal/AbstractManagedChannelImplBuilder.java b/core/src/main/java/io/grpc/internal/AbstractManagedChannelImplBuilder.java deleted file mode 100644 index aac6c25a8ee..00000000000 --- a/core/src/main/java/io/grpc/internal/AbstractManagedChannelImplBuilder.java +++ /dev/null @@ -1,670 +0,0 @@ -/* - * Copyright 2014 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.internal; - -import static com.google.common.base.Preconditions.checkArgument; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.MoreExecutors; -import io.grpc.Attributes; -import io.grpc.BinaryLog; -import io.grpc.ClientInterceptor; -import io.grpc.CompressorRegistry; -import io.grpc.DecompressorRegistry; -import io.grpc.EquivalentAddressGroup; -import io.grpc.InternalChannelz; -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.NameResolver; -import io.grpc.NameResolverRegistry; -import io.grpc.ProxyDetector; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.net.SocketAddress; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; -import javax.annotation.Nullable; - -/** - * Abstract base class for channel builders. - * - * @param The concrete type of this builder. - */ -public abstract class AbstractManagedChannelImplBuilder - > extends ManagedChannelBuilder { - private static final String DIRECT_ADDRESS_SCHEME = "directaddress"; - - private static final Logger log = - Logger.getLogger(AbstractManagedChannelImplBuilder.class.getName()); - - public static ManagedChannelBuilder forAddress(String name, int port) { - throw new UnsupportedOperationException("Subclass failed to hide static factory"); - } - - public static ManagedChannelBuilder forTarget(String target) { - throw new UnsupportedOperationException("Subclass failed to hide static factory"); - } - - /** - * An idle timeout larger than this would disable idle mode. - */ - @VisibleForTesting - static final long IDLE_MODE_MAX_TIMEOUT_DAYS = 30; - - /** - * The default idle timeout. - */ - @VisibleForTesting - static final long IDLE_MODE_DEFAULT_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(30); - - /** - * An idle timeout smaller than this would be capped to it. - */ - static final long IDLE_MODE_MIN_TIMEOUT_MILLIS = TimeUnit.SECONDS.toMillis(1); - - private static final ObjectPool DEFAULT_EXECUTOR_POOL = - SharedResourcePool.forResource(GrpcUtil.SHARED_CHANNEL_EXECUTOR); - - private static final DecompressorRegistry DEFAULT_DECOMPRESSOR_REGISTRY = - DecompressorRegistry.getDefaultInstance(); - - private static final CompressorRegistry DEFAULT_COMPRESSOR_REGISTRY = - CompressorRegistry.getDefaultInstance(); - - private static final long DEFAULT_RETRY_BUFFER_SIZE_IN_BYTES = 1L << 24; // 16M - private static final long DEFAULT_PER_RPC_BUFFER_LIMIT_IN_BYTES = 1L << 20; // 1M - - ObjectPool executorPool = DEFAULT_EXECUTOR_POOL; - - ObjectPool offloadExecutorPool = DEFAULT_EXECUTOR_POOL; - - private final List interceptors = new ArrayList<>(); - final NameResolverRegistry nameResolverRegistry = NameResolverRegistry.getDefaultRegistry(); - - // Access via getter, which may perform authority override as needed - private NameResolver.Factory nameResolverFactory = nameResolverRegistry.asFactory(); - - final String target; - - @Nullable - private final SocketAddress directServerAddress; - - @Nullable - String userAgent; - - @VisibleForTesting - @Nullable - String authorityOverride; - - String defaultLbPolicy = GrpcUtil.DEFAULT_LB_POLICY; - - boolean fullStreamDecompression; - - DecompressorRegistry decompressorRegistry = DEFAULT_DECOMPRESSOR_REGISTRY; - - CompressorRegistry compressorRegistry = DEFAULT_COMPRESSOR_REGISTRY; - - long idleTimeoutMillis = IDLE_MODE_DEFAULT_TIMEOUT_MILLIS; - - int maxRetryAttempts = 5; - int maxHedgedAttempts = 5; - long retryBufferSize = DEFAULT_RETRY_BUFFER_SIZE_IN_BYTES; - long perRpcBufferLimit = DEFAULT_PER_RPC_BUFFER_LIMIT_IN_BYTES; - boolean retryEnabled = false; // TODO(zdapeng): default to true - // Temporarily disable retry when stats or tracing is enabled to avoid breakage, until we know - // what should be the desired behavior for retry + stats/tracing. - // TODO(zdapeng): delete me - boolean temporarilyDisableRetry; - - InternalChannelz channelz = InternalChannelz.instance(); - int maxTraceEvents; - - @Nullable - Map defaultServiceConfig; - boolean lookUpServiceConfig = true; - - protected TransportTracer.Factory transportTracerFactory = TransportTracer.getDefaultFactory(); - - private int maxInboundMessageSize = GrpcUtil.DEFAULT_MAX_MESSAGE_SIZE; - - @Nullable - BinaryLog binlog; - - @Nullable - ProxyDetector proxyDetector; - - /** - * Sets the maximum message size allowed for a single gRPC frame. If an inbound messages - * larger than this limit is received it will not be processed and the RPC will fail with - * RESOURCE_EXHAUSTED. - */ - // Can be overridden by subclasses. - @Override - public T maxInboundMessageSize(int max) { - checkArgument(max >= 0, "negative max"); - maxInboundMessageSize = max; - return thisT(); - } - - protected final int maxInboundMessageSize() { - return maxInboundMessageSize; - } - - private boolean statsEnabled = true; - private boolean recordStartedRpcs = true; - private boolean recordFinishedRpcs = true; - private boolean recordRealTimeMetrics = false; - private boolean tracingEnabled = true; - - protected AbstractManagedChannelImplBuilder(String target) { - this.target = Preconditions.checkNotNull(target, "target"); - this.directServerAddress = null; - } - - /** - * Returns a target string for the SocketAddress. It is only used as a placeholder, because - * DirectAddressNameResolverFactory will not actually try to use it. However, it must be a valid - * URI. - */ - @VisibleForTesting - static String makeTargetStringForDirectAddress(SocketAddress address) { - try { - return new URI(DIRECT_ADDRESS_SCHEME, "", "/" + address, null).toString(); - } catch (URISyntaxException e) { - // It should not happen. - throw new RuntimeException(e); - } - } - - protected AbstractManagedChannelImplBuilder(SocketAddress directServerAddress, String authority) { - this.target = makeTargetStringForDirectAddress(directServerAddress); - this.directServerAddress = directServerAddress; - this.nameResolverFactory = new DirectAddressNameResolverFactory(directServerAddress, authority); - } - - @Override - public final T directExecutor() { - return executor(MoreExecutors.directExecutor()); - } - - @Override - public final T executor(Executor executor) { - if (executor != null) { - this.executorPool = new FixedObjectPool<>(executor); - } else { - this.executorPool = DEFAULT_EXECUTOR_POOL; - } - return thisT(); - } - - @Override - public final T offloadExecutor(Executor executor) { - if (executor != null) { - this.offloadExecutorPool = new FixedObjectPool<>(executor); - } else { - this.offloadExecutorPool = DEFAULT_EXECUTOR_POOL; - } - return thisT(); - } - - @Override - public final T intercept(List interceptors) { - this.interceptors.addAll(interceptors); - return thisT(); - } - - @Override - public final T intercept(ClientInterceptor... interceptors) { - return intercept(Arrays.asList(interceptors)); - } - - @Deprecated - @Override - public final T nameResolverFactory(NameResolver.Factory resolverFactory) { - Preconditions.checkState(directServerAddress == null, - "directServerAddress is set (%s), which forbids the use of NameResolverFactory", - directServerAddress); - if (resolverFactory != null) { - this.nameResolverFactory = resolverFactory; - } else { - this.nameResolverFactory = nameResolverRegistry.asFactory(); - } - return thisT(); - } - - @Override - public final T defaultLoadBalancingPolicy(String policy) { - Preconditions.checkState(directServerAddress == null, - "directServerAddress is set (%s), which forbids the use of load-balancing policy", - directServerAddress); - Preconditions.checkArgument(policy != null, "policy cannot be null"); - this.defaultLbPolicy = policy; - return thisT(); - } - - @Override - public final T enableFullStreamDecompression() { - this.fullStreamDecompression = true; - return thisT(); - } - - @Override - public final T decompressorRegistry(DecompressorRegistry registry) { - if (registry != null) { - this.decompressorRegistry = registry; - } else { - this.decompressorRegistry = DEFAULT_DECOMPRESSOR_REGISTRY; - } - return thisT(); - } - - @Override - public final T compressorRegistry(CompressorRegistry registry) { - if (registry != null) { - this.compressorRegistry = registry; - } else { - this.compressorRegistry = DEFAULT_COMPRESSOR_REGISTRY; - } - return thisT(); - } - - @Override - public final T userAgent(@Nullable String userAgent) { - this.userAgent = userAgent; - return thisT(); - } - - @Override - public final T overrideAuthority(String authority) { - this.authorityOverride = checkAuthority(authority); - return thisT(); - } - - @Override - public final T idleTimeout(long value, TimeUnit unit) { - checkArgument(value > 0, "idle timeout is %s, but must be positive", value); - // We convert to the largest unit to avoid overflow - if (unit.toDays(value) >= IDLE_MODE_MAX_TIMEOUT_DAYS) { - // This disables idle mode - this.idleTimeoutMillis = ManagedChannelImpl.IDLE_TIMEOUT_MILLIS_DISABLE; - } else { - this.idleTimeoutMillis = Math.max(unit.toMillis(value), IDLE_MODE_MIN_TIMEOUT_MILLIS); - } - return thisT(); - } - - @Override - public final T maxRetryAttempts(int maxRetryAttempts) { - this.maxRetryAttempts = maxRetryAttempts; - return thisT(); - } - - @Override - public final T maxHedgedAttempts(int maxHedgedAttempts) { - this.maxHedgedAttempts = maxHedgedAttempts; - return thisT(); - } - - @Override - public final T retryBufferSize(long bytes) { - checkArgument(bytes > 0L, "retry buffer size must be positive"); - retryBufferSize = bytes; - return thisT(); - } - - @Override - public final T perRpcBufferLimit(long bytes) { - checkArgument(bytes > 0L, "per RPC buffer limit must be positive"); - perRpcBufferLimit = bytes; - return thisT(); - } - - @Override - public final T disableRetry() { - retryEnabled = false; - return thisT(); - } - - @Override - public final T enableRetry() { - retryEnabled = true; - statsEnabled = false; - tracingEnabled = false; - return thisT(); - } - - @Override - public final T setBinaryLog(BinaryLog binlog) { - this.binlog = binlog; - return thisT(); - } - - @Override - public T maxTraceEvents(int maxTraceEvents) { - checkArgument(maxTraceEvents >= 0, "maxTraceEvents must be non-negative"); - this.maxTraceEvents = maxTraceEvents; - return thisT(); - } - - @Override - public T proxyDetector(@Nullable ProxyDetector proxyDetector) { - this.proxyDetector = proxyDetector; - return thisT(); - } - - @Override - public T defaultServiceConfig(@Nullable Map serviceConfig) { - // TODO(notcarl): use real parsing - defaultServiceConfig = checkMapEntryTypes(serviceConfig); - return thisT(); - } - - @Nullable - private static Map checkMapEntryTypes(@Nullable Map map) { - if (map == null) { - return null; - } - // Not using ImmutableMap.Builder because of extra guava dependency for Android. - Map parsedMap = new LinkedHashMap<>(); - for (Map.Entry entry : map.entrySet()) { - checkArgument( - entry.getKey() instanceof String, - "The key of the entry '%s' is not of String type", entry); - - String key = (String) entry.getKey(); - Object value = entry.getValue(); - if (value == null) { - parsedMap.put(key, null); - } else if (value instanceof Map) { - parsedMap.put(key, checkMapEntryTypes((Map) value)); - } else if (value instanceof List) { - parsedMap.put(key, checkListEntryTypes((List) value)); - } else if (value instanceof String) { - parsedMap.put(key, value); - } else if (value instanceof Double) { - parsedMap.put(key, value); - } else if (value instanceof Boolean) { - parsedMap.put(key, value); - } else { - throw new IllegalArgumentException( - "The value of the map entry '" + entry + "' is of type '" + value.getClass() - + "', which is not supported"); - } - } - return Collections.unmodifiableMap(parsedMap); - } - - private static List checkListEntryTypes(List list) { - List parsedList = new ArrayList<>(list.size()); - for (Object value : list) { - if (value == null) { - parsedList.add(null); - } else if (value instanceof Map) { - parsedList.add(checkMapEntryTypes((Map) value)); - } else if (value instanceof List) { - parsedList.add(checkListEntryTypes((List) value)); - } else if (value instanceof String) { - parsedList.add(value); - } else if (value instanceof Double) { - parsedList.add(value); - } else if (value instanceof Boolean) { - parsedList.add(value); - } else { - throw new IllegalArgumentException( - "The entry '" + value + "' is of type '" + value.getClass() - + "', which is not supported"); - } - } - return Collections.unmodifiableList(parsedList); - } - - @Override - public T disableServiceConfigLookUp() { - this.lookUpServiceConfig = false; - return thisT(); - } - - /** - * Disable or enable stats features. Enabled by default. - * - *

For the current release, calling {@code setStatsEnabled(true)} may have a side effect that - * disables retry. - */ - protected void setStatsEnabled(boolean value) { - statsEnabled = value; - } - - /** - * Disable or enable stats recording for RPC upstarts. Effective only if {@link - * #setStatsEnabled} is set to true. Enabled by default. - */ - protected void setStatsRecordStartedRpcs(boolean value) { - recordStartedRpcs = value; - } - - /** - * Disable or enable stats recording for RPC completions. Effective only if {@link - * #setStatsEnabled} is set to true. Enabled by default. - */ - protected void setStatsRecordFinishedRpcs(boolean value) { - recordFinishedRpcs = value; - } - - /** - * Disable or enable real-time metrics recording. Effective only if {@link #setStatsEnabled} is - * set to true. Disabled by default. - */ - protected void setStatsRecordRealTimeMetrics(boolean value) { - recordRealTimeMetrics = value; - } - - /** - * Disable or enable tracing features. Enabled by default. - * - *

For the current release, calling {@code setTracingEnabled(true)} may have a side effect that - * disables retry. - */ - protected void setTracingEnabled(boolean value) { - tracingEnabled = value; - } - - @VisibleForTesting - final long getIdleTimeoutMillis() { - return idleTimeoutMillis; - } - - /** - * Verifies the authority is valid. This method exists as an escape hatch for putting in an - * authority that is valid, but would fail the default validation provided by this - * implementation. - */ - protected String checkAuthority(String authority) { - return GrpcUtil.checkAuthority(authority); - } - - @Override - public ManagedChannel build() { - return new ManagedChannelOrphanWrapper(new ManagedChannelImpl( - this, - buildTransportFactory(), - new ExponentialBackoffPolicy.Provider(), - SharedResourcePool.forResource(GrpcUtil.SHARED_CHANNEL_EXECUTOR), - GrpcUtil.STOPWATCH_SUPPLIER, - getEffectiveInterceptors(), - TimeProvider.SYSTEM_TIME_PROVIDER)); - } - - // Temporarily disable retry when stats or tracing is enabled to avoid breakage, until we know - // what should be the desired behavior for retry + stats/tracing. - // TODO(zdapeng): FIX IT - @VisibleForTesting - final List getEffectiveInterceptors() { - List effectiveInterceptors = - new ArrayList<>(this.interceptors); - temporarilyDisableRetry = false; - if (statsEnabled) { - temporarilyDisableRetry = true; - ClientInterceptor statsInterceptor = null; - try { - Class censusStatsAccessor = - Class.forName("io.grpc.census.InternalCensusStatsAccessor"); - Method getClientInterceptorMethod = - censusStatsAccessor.getDeclaredMethod( - "getClientInterceptor", - boolean.class, - boolean.class, - boolean.class); - statsInterceptor = - (ClientInterceptor) getClientInterceptorMethod - .invoke( - null, - recordStartedRpcs, - recordFinishedRpcs, - recordRealTimeMetrics); - } catch (ClassNotFoundException e) { - // Replace these separate catch statements with multicatch when Android min-API >= 19 - log.log(Level.FINE, "Unable to apply census stats", e); - } catch (NoSuchMethodException e) { - log.log(Level.FINE, "Unable to apply census stats", e); - } catch (IllegalAccessException e) { - log.log(Level.FINE, "Unable to apply census stats", e); - } catch (InvocationTargetException e) { - log.log(Level.FINE, "Unable to apply census stats", e); - } - if (statsInterceptor != null) { - // First interceptor runs last (see ClientInterceptors.intercept()), so that no - // other interceptor can override the tracer factory we set in CallOptions. - effectiveInterceptors.add(0, statsInterceptor); - } - } - if (tracingEnabled) { - temporarilyDisableRetry = true; - ClientInterceptor tracingInterceptor = null; - try { - Class censusTracingAccessor = - Class.forName("io.grpc.census.InternalCensusTracingAccessor"); - Method getClientInterceptroMethod = - censusTracingAccessor.getDeclaredMethod("getClientInterceptor"); - tracingInterceptor = (ClientInterceptor) getClientInterceptroMethod.invoke(null); - } catch (ClassNotFoundException e) { - // Replace these separate catch statements with multicatch when Android min-API >= 19 - log.log(Level.FINE, "Unable to apply census stats", e); - } catch (NoSuchMethodException e) { - log.log(Level.FINE, "Unable to apply census stats", e); - } catch (IllegalAccessException e) { - log.log(Level.FINE, "Unable to apply census stats", e); - } catch (InvocationTargetException e) { - log.log(Level.FINE, "Unable to apply census stats", e); - } - if (tracingInterceptor != null) { - effectiveInterceptors.add(0, tracingInterceptor); - } - } - return effectiveInterceptors; - } - - /** - * Subclasses should override this method to provide the {@link ClientTransportFactory} - * appropriate for this channel. This method is meant for Transport implementors and should not - * be used by normal users. - */ - protected abstract ClientTransportFactory buildTransportFactory(); - - /** - * Subclasses can override this method to provide a default port to {@link NameResolver} for use - * in cases where the target string doesn't include a port. The default implementation returns - * {@link GrpcUtil#DEFAULT_PORT_SSL}. - */ - protected int getDefaultPort() { - return GrpcUtil.DEFAULT_PORT_SSL; - } - - /** - * Returns a {@link NameResolver.Factory} for the channel. - */ - NameResolver.Factory getNameResolverFactory() { - if (authorityOverride == null) { - return nameResolverFactory; - } else { - return new OverrideAuthorityNameResolverFactory(nameResolverFactory, authorityOverride); - } - } - - private static class DirectAddressNameResolverFactory extends NameResolver.Factory { - final SocketAddress address; - final String authority; - - DirectAddressNameResolverFactory(SocketAddress address, String authority) { - this.address = address; - this.authority = authority; - } - - @Override - public NameResolver newNameResolver(URI notUsedUri, NameResolver.Args args) { - return new NameResolver() { - @Override - public String getServiceAuthority() { - return authority; - } - - @Override - public void start(Listener2 listener) { - listener.onResult( - ResolutionResult.newBuilder() - .setAddresses(Collections.singletonList(new EquivalentAddressGroup(address))) - .setAttributes(Attributes.EMPTY) - .build()); - } - - @Override - public void shutdown() {} - }; - } - - @Override - public String getDefaultScheme() { - return DIRECT_ADDRESS_SCHEME; - } - } - - /** - * Returns the correctly typed version of the builder. - */ - private T thisT() { - @SuppressWarnings("unchecked") - T thisT = (T) this; - return thisT; - } - - /** - * Returns the internal offload executor pool for offloading tasks. - */ - protected ObjectPool getOffloadExecutorPool() { - return this.offloadExecutorPool; - } -} diff --git a/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java b/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java index 1bd42c04e8d..d2bc87cdde3 100644 --- a/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java +++ b/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java @@ -44,6 +44,7 @@ import io.grpc.Context; import io.grpc.DecompressorRegistry; import io.grpc.EquivalentAddressGroup; +import io.grpc.ForwardingChannelBuilder; import io.grpc.InternalChannelz; import io.grpc.InternalChannelz.ChannelStats; import io.grpc.InternalChannelz.ChannelTrace; @@ -72,6 +73,8 @@ import io.grpc.SynchronizationContext.ScheduledHandle; import io.grpc.internal.AutoConfiguredLoadBalancerFactory.AutoConfiguredLoadBalancer; import io.grpc.internal.ClientCallImpl.ClientStreamProvider; +import io.grpc.internal.ManagedChannelImplBuilder.FixedPortProvider; +import io.grpc.internal.ManagedChannelImplBuilder.UnsupportedClientTransportFactoryBuilder; import io.grpc.internal.ManagedChannelServiceConfig.MethodInfo; import io.grpc.internal.RetriableStream.ChannelBufferMeter; import io.grpc.internal.RetriableStream.Throttle; @@ -574,7 +577,7 @@ ClientStream newSubstream(ClientStreamTracer.Factory tracerFactory, Metadata new private final Rescheduler idleTimer; ManagedChannelImpl( - AbstractManagedChannelImplBuilder builder, + ManagedChannelImplBuilder builder, ClientTransportFactory clientTransportFactory, BackoffPolicy.Provider backoffPolicyProvider, ObjectPool balancerRpcExecutorPool, @@ -661,7 +664,7 @@ public void execute(Runnable command) { } else { checkArgument( builder.idleTimeoutMillis - >= AbstractManagedChannelImplBuilder.IDLE_MODE_MIN_TIMEOUT_MILLIS, + >= ManagedChannelImplBuilder.IDLE_MODE_MIN_TIMEOUT_MILLIS, "invalid idleTimeoutMillis %s", builder.idleTimeoutMillis); this.idleTimeoutMillis = builder.idleTimeoutMillis; } @@ -1446,28 +1449,27 @@ public void run() { @Override public ManagedChannelBuilder createResolvingOobChannelBuilder(String target) { final class ResolvingOobChannelBuilder - extends AbstractManagedChannelImplBuilder { - int defaultPort = -1; + extends ForwardingChannelBuilder { + private final ManagedChannelImplBuilder managedChannelImplBuilder; ResolvingOobChannelBuilder(String target) { - super(target); + managedChannelImplBuilder = new ManagedChannelImplBuilder(target, + new UnsupportedClientTransportFactoryBuilder(), + new FixedPortProvider(nameResolverArgs.getDefaultPort())); + managedChannelImplBuilder.executorPool = executorPool; + managedChannelImplBuilder.offloadExecutorPool = offloadExecutorHolder.pool; } @Override - public int getDefaultPort() { - return defaultPort; - } - - @Override - protected ClientTransportFactory buildTransportFactory() { - throw new UnsupportedOperationException(); + protected ManagedChannelBuilder delegate() { + return managedChannelImplBuilder; } @Override public ManagedChannel build() { // TODO(creamsoup) prevent main channel to shutdown if oob channel is not terminated return new ManagedChannelImpl( - this, + managedChannelImplBuilder, transportFactory, backoffPolicyProvider, balancerRpcExecutorPool, @@ -1479,17 +1481,15 @@ public ManagedChannel build() { checkState(!terminated, "Channel is terminated"); - ResolvingOobChannelBuilder builder = new ResolvingOobChannelBuilder(target); - builder.offloadExecutorPool = offloadExecutorHolder.pool; - builder.overrideAuthority(getAuthority()); @SuppressWarnings("deprecation") - ResolvingOobChannelBuilder unused = builder.nameResolverFactory(nameResolverFactory); - builder.executorPool = executorPool; - builder.maxTraceEvents = maxTraceEvents; - builder.proxyDetector = nameResolverArgs.getProxyDetector(); - builder.defaultPort = nameResolverArgs.getDefaultPort(); - builder.userAgent = userAgent; - return builder; + ResolvingOobChannelBuilder builder = new ResolvingOobChannelBuilder(target) + .nameResolverFactory(nameResolverFactory); + + return builder + .overrideAuthority(getAuthority()) + .maxTraceEvents(maxTraceEvents) + .proxyDetector(nameResolverArgs.getProxyDetector()) + .userAgent(userAgent); } @Override diff --git a/core/src/main/java/io/grpc/internal/ManagedChannelImplBuilder.java b/core/src/main/java/io/grpc/internal/ManagedChannelImplBuilder.java index d2807df200b..202055f9714 100644 --- a/core/src/main/java/io/grpc/internal/ManagedChannelImplBuilder.java +++ b/core/src/main/java/io/grpc/internal/ManagedChannelImplBuilder.java @@ -16,19 +16,148 @@ package io.grpc.internal; +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.util.concurrent.MoreExecutors; +import io.grpc.Attributes; +import io.grpc.BinaryLog; +import io.grpc.ClientInterceptor; +import io.grpc.CompressorRegistry; +import io.grpc.DecompressorRegistry; +import io.grpc.EquivalentAddressGroup; +import io.grpc.InternalChannelz; +import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; +import io.grpc.NameResolver; +import io.grpc.NameResolverRegistry; +import io.grpc.ProxyDetector; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.net.SocketAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.annotation.Nullable; /** * Default managed channel builder, for usage in Transport implementations. */ public final class ManagedChannelImplBuilder - extends AbstractManagedChannelImplBuilder { + extends ManagedChannelBuilder { + private static final String DIRECT_ADDRESS_SCHEME = "directaddress"; + + private static final Logger log = Logger.getLogger(ManagedChannelImplBuilder.class.getName()); + + public static ManagedChannelBuilder forAddress(String name, int port) { + throw new UnsupportedOperationException( + "ClientTransportFactoryBuilder is required, use a constructor"); + } + + public static ManagedChannelBuilder forTarget(String target) { + throw new UnsupportedOperationException( + "ClientTransportFactoryBuilder is required, use a constructor"); + } + + /** + * An idle timeout larger than this would disable idle mode. + */ + @VisibleForTesting + static final long IDLE_MODE_MAX_TIMEOUT_DAYS = 30; + + /** + * The default idle timeout. + */ + @VisibleForTesting + static final long IDLE_MODE_DEFAULT_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(30); + + /** + * An idle timeout smaller than this would be capped to it. + */ + static final long IDLE_MODE_MIN_TIMEOUT_MILLIS = TimeUnit.SECONDS.toMillis(1); + + private static final ObjectPool DEFAULT_EXECUTOR_POOL = + SharedResourcePool.forResource(GrpcUtil.SHARED_CHANNEL_EXECUTOR); + + private static final DecompressorRegistry DEFAULT_DECOMPRESSOR_REGISTRY = + DecompressorRegistry.getDefaultInstance(); + + private static final CompressorRegistry DEFAULT_COMPRESSOR_REGISTRY = + CompressorRegistry.getDefaultInstance(); + + private static final long DEFAULT_RETRY_BUFFER_SIZE_IN_BYTES = 1L << 24; // 16M + private static final long DEFAULT_PER_RPC_BUFFER_LIMIT_IN_BYTES = 1L << 20; // 1M + + ObjectPool executorPool = DEFAULT_EXECUTOR_POOL; + + ObjectPool offloadExecutorPool = DEFAULT_EXECUTOR_POOL; + + private final List interceptors = new ArrayList<>(); + final NameResolverRegistry nameResolverRegistry = NameResolverRegistry.getDefaultRegistry(); + + // Access via getter, which may perform authority override as needed + private NameResolver.Factory nameResolverFactory = nameResolverRegistry.asFactory(); + + final String target; + + @Nullable + private final SocketAddress directServerAddress; + + @Nullable + String userAgent; + + @Nullable + private String authorityOverride; + + String defaultLbPolicy = GrpcUtil.DEFAULT_LB_POLICY; + + boolean fullStreamDecompression; + + DecompressorRegistry decompressorRegistry = DEFAULT_DECOMPRESSOR_REGISTRY; + + CompressorRegistry compressorRegistry = DEFAULT_COMPRESSOR_REGISTRY; + + long idleTimeoutMillis = IDLE_MODE_DEFAULT_TIMEOUT_MILLIS; + + int maxRetryAttempts = 5; + int maxHedgedAttempts = 5; + long retryBufferSize = DEFAULT_RETRY_BUFFER_SIZE_IN_BYTES; + long perRpcBufferLimit = DEFAULT_PER_RPC_BUFFER_LIMIT_IN_BYTES; + boolean retryEnabled = false; // TODO(zdapeng): default to true + // Temporarily disable retry when stats or tracing is enabled to avoid breakage, until we know + // what should be the desired behavior for retry + stats/tracing. + // TODO(zdapeng): delete me + boolean temporarilyDisableRetry; + + InternalChannelz channelz = InternalChannelz.instance(); + int maxTraceEvents; + + @Nullable + Map defaultServiceConfig; + boolean lookUpServiceConfig = true; + + @Nullable + BinaryLog binlog; + + @Nullable + ProxyDetector proxyDetector; private boolean authorityCheckerDisabled; + private boolean statsEnabled = true; + private boolean recordStartedRpcs = true; + private boolean recordFinishedRpcs = true; + private boolean recordRealTimeMetrics = false; + private boolean tracingEnabled = true; /** * An interface for Transport implementors to provide the {@link ClientTransportFactory} @@ -38,6 +167,17 @@ public interface ClientTransportFactoryBuilder { ClientTransportFactory buildClientTransportFactory(); } + /** + * Convenience ClientTransportFactoryBuilder, throws UnsupportedOperationException(). + */ + public static class UnsupportedClientTransportFactoryBuilder implements + ClientTransportFactoryBuilder { + @Override + public ClientTransportFactory buildClientTransportFactory() { + throw new UnsupportedOperationException(); + } + } + /** * An interface for Transport implementors to provide a default port to {@link * io.grpc.NameResolver} for use in cases where the target string doesn't include a port. The @@ -63,11 +203,11 @@ public int getDefaultPort() { } } - private final class ManagedChannelDefaultPortProvider implements + private static final class ManagedChannelDefaultPortProvider implements ChannelBuilderDefaultPortProvider { @Override public int getDefaultPort() { - return ManagedChannelImplBuilder.super.getDefaultPort(); + return GrpcUtil.DEFAULT_PORT_SSL; } } @@ -82,9 +222,10 @@ public int getDefaultPort() { public ManagedChannelImplBuilder(String target, ClientTransportFactoryBuilder clientTransportFactoryBuilder, @Nullable ChannelBuilderDefaultPortProvider channelBuilderDefaultPortProvider) { - super(target); + this.target = Preconditions.checkNotNull(target, "target"); this.clientTransportFactoryBuilder = Preconditions .checkNotNull(clientTransportFactoryBuilder, "clientTransportFactoryBuilder"); + this.directServerAddress = null; if (channelBuilderDefaultPortProvider != null) { this.channelBuilderDefaultPortProvider = channelBuilderDefaultPortProvider; @@ -93,6 +234,21 @@ public ManagedChannelImplBuilder(String target, } } + /** + * Returns a target string for the SocketAddress. It is only used as a placeholder, because + * DirectAddressNameResolverFactory will not actually try to use it. However, it must be a valid + * URI. + */ + @VisibleForTesting + static String makeTargetStringForDirectAddress(SocketAddress address) { + try { + return new URI(DIRECT_ADDRESS_SCHEME, "", "/" + address, null).toString(); + } catch (URISyntaxException e) { + // It should not happen. + throw new RuntimeException(e); + } + } + /** * Creates a new managed channel builder with the given server address, authority string of the * channel. Transport implementors must provide client transport factory builder, and may set @@ -101,9 +257,11 @@ public ManagedChannelImplBuilder(String target, public ManagedChannelImplBuilder(SocketAddress directServerAddress, String authority, ClientTransportFactoryBuilder clientTransportFactoryBuilder, @Nullable ChannelBuilderDefaultPortProvider channelBuilderDefaultPortProvider) { - super(directServerAddress, authority); + this.target = makeTargetStringForDirectAddress(directServerAddress); this.clientTransportFactoryBuilder = Preconditions .checkNotNull(clientTransportFactoryBuilder, "clientTransportFactoryBuilder"); + this.directServerAddress = directServerAddress; + this.nameResolverFactory = new DirectAddressNameResolverFactory(directServerAddress, authority); if (channelBuilderDefaultPortProvider != null) { this.channelBuilderDefaultPortProvider = channelBuilderDefaultPortProvider; @@ -113,70 +271,461 @@ public ManagedChannelImplBuilder(SocketAddress directServerAddress, String autho } @Override - protected ClientTransportFactory buildTransportFactory() { - return clientTransportFactoryBuilder.buildClientTransportFactory(); + public ManagedChannelImplBuilder directExecutor() { + return executor(MoreExecutors.directExecutor()); } @Override - protected int getDefaultPort() { - return channelBuilderDefaultPortProvider.getDefaultPort(); + public ManagedChannelImplBuilder executor(Executor executor) { + if (executor != null) { + this.executorPool = new FixedObjectPool<>(executor); + } else { + this.executorPool = DEFAULT_EXECUTOR_POOL; + } + return this; } - /** Disable the check whether the authority is valid. */ - public ManagedChannelImplBuilder disableCheckAuthority() { - authorityCheckerDisabled = true; + @Override + public ManagedChannelImplBuilder offloadExecutor(Executor executor) { + if (executor != null) { + this.offloadExecutorPool = new FixedObjectPool<>(executor); + } else { + this.offloadExecutorPool = DEFAULT_EXECUTOR_POOL; + } return this; } - /** Enable previously disabled authority check. */ - public ManagedChannelImplBuilder enableCheckAuthority() { - authorityCheckerDisabled = false; + @Override + public ManagedChannelImplBuilder intercept(List interceptors) { + this.interceptors.addAll(interceptors); return this; } @Override - protected String checkAuthority(String authority) { - if (authorityCheckerDisabled) { - return authority; + public ManagedChannelImplBuilder intercept(ClientInterceptor... interceptors) { + return intercept(Arrays.asList(interceptors)); + } + + @Deprecated + @Override + public ManagedChannelImplBuilder nameResolverFactory(NameResolver.Factory resolverFactory) { + Preconditions.checkState(directServerAddress == null, + "directServerAddress is set (%s), which forbids the use of NameResolverFactory", + directServerAddress); + if (resolverFactory != null) { + this.nameResolverFactory = resolverFactory; + } else { + this.nameResolverFactory = nameResolverRegistry.asFactory(); } - return super.checkAuthority(authority); + return this; } @Override - public void setStatsEnabled(boolean value) { - super.setStatsEnabled(value); + public ManagedChannelImplBuilder defaultLoadBalancingPolicy(String policy) { + Preconditions.checkState(directServerAddress == null, + "directServerAddress is set (%s), which forbids the use of load-balancing policy", + directServerAddress); + Preconditions.checkArgument(policy != null, "policy cannot be null"); + this.defaultLbPolicy = policy; + return this; } @Override - public void setStatsRecordStartedRpcs(boolean value) { - super.setStatsRecordStartedRpcs(value); + public ManagedChannelImplBuilder enableFullStreamDecompression() { + this.fullStreamDecompression = true; + return this; } @Override - public void setStatsRecordFinishedRpcs(boolean value) { - super.setStatsRecordFinishedRpcs(value); + public ManagedChannelImplBuilder decompressorRegistry(DecompressorRegistry registry) { + if (registry != null) { + this.decompressorRegistry = registry; + } else { + this.decompressorRegistry = DEFAULT_DECOMPRESSOR_REGISTRY; + } + return this; } @Override - public void setStatsRecordRealTimeMetrics(boolean value) { - super.setStatsRecordRealTimeMetrics(value); + public ManagedChannelImplBuilder compressorRegistry(CompressorRegistry registry) { + if (registry != null) { + this.compressorRegistry = registry; + } else { + this.compressorRegistry = DEFAULT_COMPRESSOR_REGISTRY; + } + return this; + } + + @Override + public ManagedChannelImplBuilder userAgent(@Nullable String userAgent) { + this.userAgent = userAgent; + return this; + } + + @Override + public ManagedChannelImplBuilder overrideAuthority(String authority) { + this.authorityOverride = checkAuthority(authority); + return this; + } + + @Nullable + @VisibleForTesting + String getOverrideAuthority() { + return authorityOverride; + } + + @Override + public ManagedChannelImplBuilder idleTimeout(long value, TimeUnit unit) { + checkArgument(value > 0, "idle timeout is %s, but must be positive", value); + // We convert to the largest unit to avoid overflow + if (unit.toDays(value) >= IDLE_MODE_MAX_TIMEOUT_DAYS) { + // This disables idle mode + this.idleTimeoutMillis = ManagedChannelImpl.IDLE_TIMEOUT_MILLIS_DISABLE; + } else { + this.idleTimeoutMillis = Math.max(unit.toMillis(value), IDLE_MODE_MIN_TIMEOUT_MILLIS); + } + return this; + } + + @Override + public ManagedChannelImplBuilder maxRetryAttempts(int maxRetryAttempts) { + this.maxRetryAttempts = maxRetryAttempts; + return this; + } + + @Override + public ManagedChannelImplBuilder maxHedgedAttempts(int maxHedgedAttempts) { + this.maxHedgedAttempts = maxHedgedAttempts; + return this; + } + + @Override + public ManagedChannelImplBuilder retryBufferSize(long bytes) { + checkArgument(bytes > 0L, "retry buffer size must be positive"); + retryBufferSize = bytes; + return this; } @Override + public ManagedChannelImplBuilder perRpcBufferLimit(long bytes) { + checkArgument(bytes > 0L, "per RPC buffer limit must be positive"); + perRpcBufferLimit = bytes; + return this; + } + + @Override + public ManagedChannelImplBuilder disableRetry() { + retryEnabled = false; + return this; + } + + @Override + public ManagedChannelImplBuilder enableRetry() { + retryEnabled = true; + statsEnabled = false; + tracingEnabled = false; + return this; + } + + @Override + public ManagedChannelImplBuilder setBinaryLog(BinaryLog binlog) { + this.binlog = binlog; + return this; + } + + @Override + public ManagedChannelImplBuilder maxTraceEvents(int maxTraceEvents) { + checkArgument(maxTraceEvents >= 0, "maxTraceEvents must be non-negative"); + this.maxTraceEvents = maxTraceEvents; + return this; + } + + @Override + public ManagedChannelImplBuilder proxyDetector(@Nullable ProxyDetector proxyDetector) { + this.proxyDetector = proxyDetector; + return this; + } + + @Override + public ManagedChannelImplBuilder defaultServiceConfig(@Nullable Map serviceConfig) { + // TODO(notcarl): use real parsing + defaultServiceConfig = checkMapEntryTypes(serviceConfig); + return this; + } + + @Nullable + private static Map checkMapEntryTypes(@Nullable Map map) { + if (map == null) { + return null; + } + // Not using ImmutableMap.Builder because of extra guava dependency for Android. + Map parsedMap = new LinkedHashMap<>(); + for (Map.Entry entry : map.entrySet()) { + checkArgument( + entry.getKey() instanceof String, + "The key of the entry '%s' is not of String type", entry); + + String key = (String) entry.getKey(); + Object value = entry.getValue(); + if (value == null) { + parsedMap.put(key, null); + } else if (value instanceof Map) { + parsedMap.put(key, checkMapEntryTypes((Map) value)); + } else if (value instanceof List) { + parsedMap.put(key, checkListEntryTypes((List) value)); + } else if (value instanceof String) { + parsedMap.put(key, value); + } else if (value instanceof Double) { + parsedMap.put(key, value); + } else if (value instanceof Boolean) { + parsedMap.put(key, value); + } else { + throw new IllegalArgumentException( + "The value of the map entry '" + entry + "' is of type '" + value.getClass() + + "', which is not supported"); + } + } + return Collections.unmodifiableMap(parsedMap); + } + + private static List checkListEntryTypes(List list) { + List parsedList = new ArrayList<>(list.size()); + for (Object value : list) { + if (value == null) { + parsedList.add(null); + } else if (value instanceof Map) { + parsedList.add(checkMapEntryTypes((Map) value)); + } else if (value instanceof List) { + parsedList.add(checkListEntryTypes((List) value)); + } else if (value instanceof String) { + parsedList.add(value); + } else if (value instanceof Double) { + parsedList.add(value); + } else if (value instanceof Boolean) { + parsedList.add(value); + } else { + throw new IllegalArgumentException( + "The entry '" + value + "' is of type '" + value.getClass() + + "', which is not supported"); + } + } + return Collections.unmodifiableList(parsedList); + } + + @Override + public ManagedChannelImplBuilder disableServiceConfigLookUp() { + this.lookUpServiceConfig = false; + return this; + } + + /** + * Disable or enable stats features. Enabled by default. + * + *

For the current release, calling {@code setStatsEnabled(true)} may have a side effect that + * disables retry. + */ + public void setStatsEnabled(boolean value) { + statsEnabled = value; + } + + /** + * Disable or enable stats recording for RPC upstarts. Effective only if {@link + * #setStatsEnabled} is set to true. Enabled by default. + */ + public void setStatsRecordStartedRpcs(boolean value) { + recordStartedRpcs = value; + } + + /** + * Disable or enable stats recording for RPC completions. Effective only if {@link + * #setStatsEnabled} is set to true. Enabled by default. + */ + public void setStatsRecordFinishedRpcs(boolean value) { + recordFinishedRpcs = value; + } + + /** + * Disable or enable real-time metrics recording. Effective only if {@link #setStatsEnabled} is + * set to true. Disabled by default. + */ + public void setStatsRecordRealTimeMetrics(boolean value) { + recordRealTimeMetrics = value; + } + + /** + * Disable or enable tracing features. Enabled by default. + * + *

For the current release, calling {@code setTracingEnabled(true)} may have a side effect that + * disables retry. + */ public void setTracingEnabled(boolean value) { - super.setTracingEnabled(value); + tracingEnabled = value; + } + + /** + * Verifies the authority is valid. + */ + @VisibleForTesting + String checkAuthority(String authority) { + if (authorityCheckerDisabled) { + return authority; + } + return GrpcUtil.checkAuthority(authority); + } + + /** Disable the check whether the authority is valid. */ + public ManagedChannelImplBuilder disableCheckAuthority() { + authorityCheckerDisabled = true; + return this; + } + + /** Enable previously disabled authority check. */ + public ManagedChannelImplBuilder enableCheckAuthority() { + authorityCheckerDisabled = false; + return this; } @Override - public ObjectPool getOffloadExecutorPool() { - return super.getOffloadExecutorPool(); + public ManagedChannel build() { + return new ManagedChannelOrphanWrapper(new ManagedChannelImpl( + this, + clientTransportFactoryBuilder.buildClientTransportFactory(), + new ExponentialBackoffPolicy.Provider(), + SharedResourcePool.forResource(GrpcUtil.SHARED_CHANNEL_EXECUTOR), + GrpcUtil.STOPWATCH_SUPPLIER, + getEffectiveInterceptors(), + TimeProvider.SYSTEM_TIME_PROVIDER)); } - public static ManagedChannelBuilder forAddress(String name, int port) { - throw new UnsupportedOperationException("ClientTransportFactoryBuilder is required"); + // Temporarily disable retry when stats or tracing is enabled to avoid breakage, until we know + // what should be the desired behavior for retry + stats/tracing. + // TODO(zdapeng): FIX IT + @VisibleForTesting + List getEffectiveInterceptors() { + List effectiveInterceptors = + new ArrayList<>(this.interceptors); + temporarilyDisableRetry = false; + if (statsEnabled) { + temporarilyDisableRetry = true; + ClientInterceptor statsInterceptor = null; + try { + Class censusStatsAccessor = + Class.forName("io.grpc.census.InternalCensusStatsAccessor"); + Method getClientInterceptorMethod = + censusStatsAccessor.getDeclaredMethod( + "getClientInterceptor", + boolean.class, + boolean.class, + boolean.class); + statsInterceptor = + (ClientInterceptor) getClientInterceptorMethod + .invoke( + null, + recordStartedRpcs, + recordFinishedRpcs, + recordRealTimeMetrics); + } catch (ClassNotFoundException e) { + // Replace these separate catch statements with multicatch when Android min-API >= 19 + log.log(Level.FINE, "Unable to apply census stats", e); + } catch (NoSuchMethodException e) { + log.log(Level.FINE, "Unable to apply census stats", e); + } catch (IllegalAccessException e) { + log.log(Level.FINE, "Unable to apply census stats", e); + } catch (InvocationTargetException e) { + log.log(Level.FINE, "Unable to apply census stats", e); + } + if (statsInterceptor != null) { + // First interceptor runs last (see ClientInterceptors.intercept()), so that no + // other interceptor can override the tracer factory we set in CallOptions. + effectiveInterceptors.add(0, statsInterceptor); + } + } + if (tracingEnabled) { + temporarilyDisableRetry = true; + ClientInterceptor tracingInterceptor = null; + try { + Class censusTracingAccessor = + Class.forName("io.grpc.census.InternalCensusTracingAccessor"); + Method getClientInterceptroMethod = + censusTracingAccessor.getDeclaredMethod("getClientInterceptor"); + tracingInterceptor = (ClientInterceptor) getClientInterceptroMethod.invoke(null); + } catch (ClassNotFoundException e) { + // Replace these separate catch statements with multicatch when Android min-API >= 19 + log.log(Level.FINE, "Unable to apply census stats", e); + } catch (NoSuchMethodException e) { + log.log(Level.FINE, "Unable to apply census stats", e); + } catch (IllegalAccessException e) { + log.log(Level.FINE, "Unable to apply census stats", e); + } catch (InvocationTargetException e) { + log.log(Level.FINE, "Unable to apply census stats", e); + } + if (tracingInterceptor != null) { + effectiveInterceptors.add(0, tracingInterceptor); + } + } + return effectiveInterceptors; } - public static ManagedChannelBuilder forTarget(String target) { - throw new UnsupportedOperationException("ClientTransportFactoryBuilder is required"); + /** + * Returns a default port to {@link NameResolver} for use in cases where the target string doesn't + * include a port. The default implementation returns {@link GrpcUtil#DEFAULT_PORT_SSL}. + */ + int getDefaultPort() { + return channelBuilderDefaultPortProvider.getDefaultPort(); + } + + /** + * Returns a {@link NameResolver.Factory} for the channel. + */ + NameResolver.Factory getNameResolverFactory() { + if (authorityOverride == null) { + return nameResolverFactory; + } else { + return new OverrideAuthorityNameResolverFactory(nameResolverFactory, authorityOverride); + } + } + + private static class DirectAddressNameResolverFactory extends NameResolver.Factory { + final SocketAddress address; + final String authority; + + DirectAddressNameResolverFactory(SocketAddress address, String authority) { + this.address = address; + this.authority = authority; + } + + @Override + public NameResolver newNameResolver(URI notUsedUri, NameResolver.Args args) { + return new NameResolver() { + @Override + public String getServiceAuthority() { + return authority; + } + + @Override + public void start(Listener2 listener) { + listener.onResult( + ResolutionResult.newBuilder() + .setAddresses(Collections.singletonList(new EquivalentAddressGroup(address))) + .setAttributes(Attributes.EMPTY) + .build()); + } + + @Override + public void shutdown() {} + }; + } + + @Override + public String getDefaultScheme() { + return DIRECT_ADDRESS_SCHEME; + } + } + + /** + * Returns the internal offload executor pool for offloading tasks. + */ + public ObjectPool getOffloadExecutorPool() { + return this.offloadExecutorPool; } } diff --git a/core/src/test/java/io/grpc/internal/AbstractManagedChannelImplBuilderTest.java b/core/src/test/java/io/grpc/internal/AbstractManagedChannelImplBuilderTest.java deleted file mode 100644 index 2ac49c48ce3..00000000000 --- a/core/src/test/java/io/grpc/internal/AbstractManagedChannelImplBuilderTest.java +++ /dev/null @@ -1,482 +0,0 @@ -/* - * Copyright 2016 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.internal; - -import static com.google.common.truth.Truth.assertThat; -import static junit.framework.TestCase.assertFalse; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; - -import com.google.common.util.concurrent.MoreExecutors; -import io.grpc.CallOptions; -import io.grpc.Channel; -import io.grpc.ClientCall; -import io.grpc.ClientInterceptor; -import io.grpc.CompressorRegistry; -import io.grpc.DecompressorRegistry; -import io.grpc.MethodDescriptor; -import io.grpc.NameResolver; -import java.net.InetSocketAddress; -import java.net.SocketAddress; -import java.net.URI; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Unit tests for {@link AbstractManagedChannelImplBuilder}. */ -@RunWith(JUnit4.class) -public class AbstractManagedChannelImplBuilderTest { - - @Rule - public final ExpectedException thrown = ExpectedException.none(); - - private static final ClientInterceptor DUMMY_USER_INTERCEPTOR = - new ClientInterceptor() { - @Override - public ClientCall interceptCall( - MethodDescriptor method, CallOptions callOptions, Channel next) { - return next.newCall(method, callOptions); - } - }; - - private final Builder builder = new Builder("fake"); - private final Builder directAddressBuilder = new Builder(new SocketAddress(){}, "fake"); - - @Test - public void executor_default() { - assertNotNull(builder.executorPool); - } - - @Test - public void executor_normal() { - Executor executor = mock(Executor.class); - assertEquals(builder, builder.executor(executor)); - assertEquals(executor, builder.executorPool.getObject()); - } - - @Test - public void executor_null() { - ObjectPool defaultValue = builder.executorPool; - builder.executor(mock(Executor.class)); - assertEquals(builder, builder.executor(null)); - assertEquals(defaultValue, builder.executorPool); - } - - @Test - public void directExecutor() { - assertEquals(builder, builder.directExecutor()); - assertEquals(MoreExecutors.directExecutor(), builder.executorPool.getObject()); - } - - @Test - public void offloadExecutor_normal() { - Executor executor = mock(Executor.class); - assertEquals(builder, builder.offloadExecutor(executor)); - assertEquals(executor, builder.offloadExecutorPool.getObject()); - } - - @Test - public void offloadExecutor_null() { - ObjectPool defaultValue = builder.offloadExecutorPool; - builder.offloadExecutor(mock(Executor.class)); - assertEquals(builder, builder.offloadExecutor(null)); - assertEquals(defaultValue, builder.offloadExecutorPool); - } - - @Test - public void nameResolverFactory_default() { - assertNotNull(builder.getNameResolverFactory()); - } - - @Test - @SuppressWarnings("deprecation") - public void nameResolverFactory_normal() { - NameResolver.Factory nameResolverFactory = mock(NameResolver.Factory.class); - assertEquals(builder, builder.nameResolverFactory(nameResolverFactory)); - assertEquals(nameResolverFactory, builder.getNameResolverFactory()); - } - - @Test - @SuppressWarnings("deprecation") - public void nameResolverFactory_null() { - NameResolver.Factory defaultValue = builder.getNameResolverFactory(); - builder.nameResolverFactory(mock(NameResolver.Factory.class)); - assertEquals(builder, builder.nameResolverFactory(null)); - assertEquals(defaultValue, builder.getNameResolverFactory()); - } - - @Test(expected = IllegalStateException.class) - @SuppressWarnings("deprecation") - public void nameResolverFactory_notAllowedWithDirectAddress() { - directAddressBuilder.nameResolverFactory(mock(NameResolver.Factory.class)); - } - - @Test - public void defaultLoadBalancingPolicy_default() { - assertEquals("pick_first", builder.defaultLbPolicy); - } - - @Test - public void defaultLoadBalancingPolicy_normal() { - assertEquals(builder, builder.defaultLoadBalancingPolicy("magic_balancer")); - assertEquals("magic_balancer", builder.defaultLbPolicy); - } - - @Test(expected = IllegalArgumentException.class) - public void defaultLoadBalancingPolicy_null() { - builder.defaultLoadBalancingPolicy(null); - } - - @Test(expected = IllegalStateException.class) - public void defaultLoadBalancingPolicy_notAllowedWithDirectAddress() { - directAddressBuilder.defaultLoadBalancingPolicy("magic_balancer"); - } - - @Test - public void fullStreamDecompression_default() { - assertFalse(builder.fullStreamDecompression); - } - - @Test - public void fullStreamDecompression_enabled() { - assertEquals(builder, builder.enableFullStreamDecompression()); - assertTrue(builder.fullStreamDecompression); - } - - @Test - public void decompressorRegistry_default() { - assertNotNull(builder.decompressorRegistry); - } - - @Test - public void decompressorRegistry_normal() { - DecompressorRegistry decompressorRegistry = DecompressorRegistry.emptyInstance(); - assertNotEquals(decompressorRegistry, builder.decompressorRegistry); - assertEquals(builder, builder.decompressorRegistry(decompressorRegistry)); - assertEquals(decompressorRegistry, builder.decompressorRegistry); - } - - @Test - public void decompressorRegistry_null() { - DecompressorRegistry defaultValue = builder.decompressorRegistry; - assertEquals(builder, builder.decompressorRegistry(DecompressorRegistry.emptyInstance())); - assertNotEquals(defaultValue, builder.decompressorRegistry); - builder.decompressorRegistry(null); - assertEquals(defaultValue, builder.decompressorRegistry); - } - - @Test - public void compressorRegistry_default() { - assertNotNull(builder.compressorRegistry); - } - - @Test - public void compressorRegistry_normal() { - CompressorRegistry compressorRegistry = CompressorRegistry.newEmptyInstance(); - assertNotEquals(compressorRegistry, builder.compressorRegistry); - assertEquals(builder, builder.compressorRegistry(compressorRegistry)); - assertEquals(compressorRegistry, builder.compressorRegistry); - } - - @Test - public void compressorRegistry_null() { - CompressorRegistry defaultValue = builder.compressorRegistry; - builder.compressorRegistry(CompressorRegistry.newEmptyInstance()); - assertNotEquals(defaultValue, builder.compressorRegistry); - assertEquals(builder, builder.compressorRegistry(null)); - assertEquals(defaultValue, builder.compressorRegistry); - } - - @Test - public void userAgent_default() { - assertNull(builder.userAgent); - } - - @Test - public void userAgent_normal() { - String userAgent = "user-agent/1"; - assertEquals(builder, builder.userAgent(userAgent)); - assertEquals(userAgent, builder.userAgent); - } - - @Test - public void userAgent_null() { - assertEquals(builder, builder.userAgent(null)); - assertNull(builder.userAgent); - - builder.userAgent("user-agent/1"); - builder.userAgent(null); - assertNull(builder.userAgent); - } - - @Test - public void overrideAuthority_default() { - assertNull(builder.authorityOverride); - } - - @Test - public void overrideAuthority_normal() { - String overrideAuthority = "best-authority"; - assertEquals(builder, builder.overrideAuthority(overrideAuthority)); - assertEquals(overrideAuthority, builder.authorityOverride); - } - - @Test(expected = NullPointerException.class) - public void overrideAuthority_null() { - builder.overrideAuthority(null); - } - - @Test(expected = IllegalArgumentException.class) - public void overrideAuthority_invalid() { - builder.overrideAuthority("not_allowed"); - } - - @Test - public void overrideAuthority_getNameResolverFactory() { - assertNull(builder.authorityOverride); - assertFalse(builder.getNameResolverFactory() instanceof OverrideAuthorityNameResolverFactory); - builder.overrideAuthority("google.com"); - assertTrue(builder.getNameResolverFactory() instanceof OverrideAuthorityNameResolverFactory); - } - - @Test - public void makeTargetStringForDirectAddress_scopedIpv6() throws Exception { - InetSocketAddress address = new InetSocketAddress("0:0:0:0:0:0:0:0%0", 10005); - assertEquals("/0:0:0:0:0:0:0:0%0:10005", address.toString()); - String target = AbstractManagedChannelImplBuilder.makeTargetStringForDirectAddress(address); - URI uri = new URI(target); - assertEquals("directaddress:////0:0:0:0:0:0:0:0%250:10005", target); - assertEquals(target, uri.toString()); - } - - @Test - public void getEffectiveInterceptors_default() { - builder.intercept(DUMMY_USER_INTERCEPTOR); - List effectiveInterceptors = builder.getEffectiveInterceptors(); - assertEquals(3, effectiveInterceptors.size()); - assertThat(effectiveInterceptors.get(0).getClass().getName()) - .isEqualTo("io.grpc.census.CensusTracingModule$TracingClientInterceptor"); - assertThat(effectiveInterceptors.get(1).getClass().getName()) - .isEqualTo("io.grpc.census.CensusStatsModule$StatsClientInterceptor"); - assertThat(effectiveInterceptors.get(2)).isSameInstanceAs(DUMMY_USER_INTERCEPTOR); - } - - @Test - public void getEffectiveInterceptors_disableStats() { - builder.intercept(DUMMY_USER_INTERCEPTOR); - builder.setStatsEnabled(false); - List effectiveInterceptors = builder.getEffectiveInterceptors(); - assertEquals(2, effectiveInterceptors.size()); - assertThat(effectiveInterceptors.get(0).getClass().getName()) - .isEqualTo("io.grpc.census.CensusTracingModule$TracingClientInterceptor"); - assertThat(effectiveInterceptors.get(1)).isSameInstanceAs(DUMMY_USER_INTERCEPTOR); - } - - @Test - public void getEffectiveInterceptors_disableTracing() { - builder.intercept(DUMMY_USER_INTERCEPTOR); - builder.setTracingEnabled(false); - List effectiveInterceptors = builder.getEffectiveInterceptors(); - assertEquals(2, effectiveInterceptors.size()); - assertThat(effectiveInterceptors.get(0).getClass().getName()) - .isEqualTo("io.grpc.census.CensusStatsModule$StatsClientInterceptor"); - assertThat(effectiveInterceptors.get(1)).isSameInstanceAs(DUMMY_USER_INTERCEPTOR); - } - - @Test - public void getEffectiveInterceptors_disableBoth() { - builder.intercept(DUMMY_USER_INTERCEPTOR); - builder.setStatsEnabled(false); - builder.setTracingEnabled(false); - List effectiveInterceptors = builder.getEffectiveInterceptors(); - assertThat(effectiveInterceptors).containsExactly(DUMMY_USER_INTERCEPTOR); - } - - @Test - public void idleTimeout() { - assertEquals(AbstractManagedChannelImplBuilder.IDLE_MODE_DEFAULT_TIMEOUT_MILLIS, - builder.getIdleTimeoutMillis()); - - builder.idleTimeout(Long.MAX_VALUE, TimeUnit.DAYS); - assertEquals(ManagedChannelImpl.IDLE_TIMEOUT_MILLIS_DISABLE, builder.getIdleTimeoutMillis()); - - builder.idleTimeout(AbstractManagedChannelImplBuilder.IDLE_MODE_MAX_TIMEOUT_DAYS, - TimeUnit.DAYS); - assertEquals(ManagedChannelImpl.IDLE_TIMEOUT_MILLIS_DISABLE, builder.getIdleTimeoutMillis()); - - try { - builder.idleTimeout(0, TimeUnit.SECONDS); - fail("Should throw"); - } catch (IllegalArgumentException e) { - // expected - } - - builder.idleTimeout(1, TimeUnit.NANOSECONDS); - assertEquals(AbstractManagedChannelImplBuilder.IDLE_MODE_MIN_TIMEOUT_MILLIS, - builder.getIdleTimeoutMillis()); - - builder.idleTimeout(30, TimeUnit.SECONDS); - assertEquals(TimeUnit.SECONDS.toMillis(30), builder.getIdleTimeoutMillis()); - } - - @Test - public void maxRetryAttempts() { - assertEquals(5, builder.maxRetryAttempts); - - builder.maxRetryAttempts(3); - assertEquals(3, builder.maxRetryAttempts); - } - - @Test - public void maxHedgedAttempts() { - assertEquals(5, builder.maxHedgedAttempts); - - builder.maxHedgedAttempts(3); - assertEquals(3, builder.maxHedgedAttempts); - } - - @Test - public void retryBufferSize() { - assertEquals(1L << 24, builder.retryBufferSize); - - builder.retryBufferSize(3456L); - assertEquals(3456L, builder.retryBufferSize); - } - - @Test - public void perRpcBufferLimit() { - assertEquals(1L << 20, builder.perRpcBufferLimit); - - builder.perRpcBufferLimit(3456L); - assertEquals(3456L, builder.perRpcBufferLimit); - } - - @Test - public void retryBufferSizeInvalidArg() { - thrown.expect(IllegalArgumentException.class); - builder.retryBufferSize(0L); - } - - @Test - public void perRpcBufferLimitInvalidArg() { - thrown.expect(IllegalArgumentException.class); - builder.perRpcBufferLimit(0L); - } - - @Test - public void disableRetry() { - builder.enableRetry(); - assertTrue(builder.retryEnabled); - - builder.disableRetry(); - assertFalse(builder.retryEnabled); - - builder.enableRetry(); - assertTrue(builder.retryEnabled); - - builder.disableRetry(); - assertFalse(builder.retryEnabled); - } - - @Test - public void defaultServiceConfig_nullKey() { - Map config = new HashMap<>(); - config.put(null, "val"); - - thrown.expect(IllegalArgumentException.class); - builder.defaultServiceConfig(config); - } - - @Test - public void defaultServiceConfig_intKey() { - Map subConfig = new HashMap<>(); - subConfig.put(3, "val"); - Map config = new HashMap<>(); - config.put("key", subConfig); - - thrown.expect(IllegalArgumentException.class); - builder.defaultServiceConfig(config); - } - - @Test - public void defaultServiceConfig_intValue() { - Map config = new HashMap<>(); - config.put("key", 3); - - thrown.expect(IllegalArgumentException.class); - builder.defaultServiceConfig(config); - } - - @Test - public void defaultServiceConfig_nested() { - Map config = new HashMap<>(); - List list1 = new ArrayList<>(); - list1.add(123D); - list1.add(null); - list1.add(true); - list1.add("str"); - Map map2 = new HashMap<>(); - map2.put("key2", false); - map2.put("key3", null); - map2.put("key4", Collections.singletonList("v4")); - map2.put("key4", 3.14D); - map2.put("key5", new HashMap()); - list1.add(map2); - config.put("key1", list1); - - builder.defaultServiceConfig(config); - - assertThat(builder.defaultServiceConfig).containsExactlyEntriesIn(config); - } - - @Test - public void disableNameResolverServiceConfig() { - assertThat(builder.lookUpServiceConfig).isTrue(); - - builder.disableServiceConfigLookUp(); - assertThat(builder.lookUpServiceConfig).isFalse(); - } - - static class Builder extends AbstractManagedChannelImplBuilder { - Builder(String target) { - super(target); - } - - Builder(SocketAddress directServerAddress, String authority) { - super(directServerAddress, authority); - } - - @Override - protected ClientTransportFactory buildTransportFactory() { - throw new UnsupportedOperationException(); - } - } -} diff --git a/core/src/test/java/io/grpc/internal/ManagedChannelImplBuilderTest.java b/core/src/test/java/io/grpc/internal/ManagedChannelImplBuilderTest.java index 5d3f4cf4f14..dfe9d1973d5 100644 --- a/core/src/test/java/io/grpc/internal/ManagedChannelImplBuilderTest.java +++ b/core/src/test/java/io/grpc/internal/ManagedChannelImplBuilderTest.java @@ -16,14 +16,43 @@ package io.grpc.internal; +import static com.google.common.truth.Truth.assertThat; +import static junit.framework.TestCase.assertFalse; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.common.util.concurrent.MoreExecutors; +import io.grpc.CallOptions; +import io.grpc.Channel; +import io.grpc.ClientCall; +import io.grpc.ClientInterceptor; +import io.grpc.CompressorRegistry; +import io.grpc.DecompressorRegistry; +import io.grpc.ManagedChannel; +import io.grpc.MethodDescriptor; +import io.grpc.NameResolver; import io.grpc.internal.ManagedChannelImplBuilder.ChannelBuilderDefaultPortProvider; import io.grpc.internal.ManagedChannelImplBuilder.ClientTransportFactoryBuilder; import io.grpc.internal.ManagedChannelImplBuilder.FixedPortProvider; +import io.grpc.internal.ManagedChannelImplBuilder.UnsupportedClientTransportFactoryBuilder; +import io.grpc.testing.GrpcCleanupRule; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -41,56 +70,295 @@ public class ManagedChannelImplBuilderTest { private static final String DUMMY_TARGET = "fake-target"; private static final String DUMMY_AUTHORITY_VALID = "valid:1234"; private static final String DUMMY_AUTHORITY_INVALID = "[ : : 1]"; + private static final ClientInterceptor DUMMY_USER_INTERCEPTOR = + new ClientInterceptor() { + @Override + public ClientCall interceptCall( + MethodDescriptor method, CallOptions callOptions, Channel next) { + return next.newCall(method, callOptions); + } + }; @Rule public final MockitoRule mocks = MockitoJUnit.rule(); @Rule public final ExpectedException thrown = ExpectedException.none(); + @Rule public final GrpcCleanupRule grpcCleanupRule = new GrpcCleanupRule(); + @Mock private ClientTransportFactory mockClientTransportFactory; @Mock private ClientTransportFactoryBuilder mockClientTransportFactoryBuilder; - @Mock private ChannelBuilderDefaultPortProvider mockChannelBuilderDefaultPortProvider; + private ManagedChannelImplBuilder builder; + private ManagedChannelImplBuilder directAddressBuilder; + private final FakeClock clock = new FakeClock(); + @Before public void setUp() throws Exception { builder = new ManagedChannelImplBuilder( DUMMY_TARGET, - mockClientTransportFactoryBuilder, - mockChannelBuilderDefaultPortProvider); - } - - /** Ensure buildTransportFactory() delegates to the custom implementation. */ - @Test - public void buildTransportFactory() { - final ClientTransportFactory clientTransportFactory = mock(ClientTransportFactory.class); - when(mockClientTransportFactoryBuilder.buildClientTransportFactory()) - .thenReturn(clientTransportFactory); - assertEquals(clientTransportFactory, builder.buildTransportFactory()); - verify(mockClientTransportFactoryBuilder).buildClientTransportFactory(); + new UnsupportedClientTransportFactoryBuilder(), + new FixedPortProvider(DUMMY_PORT)); + directAddressBuilder = new ManagedChannelImplBuilder( + new SocketAddress() {}, + DUMMY_TARGET, + new UnsupportedClientTransportFactoryBuilder(), + new FixedPortProvider(DUMMY_PORT)); } /** Ensure getDefaultPort() returns default port when no custom implementation provided. */ @Test public void getDefaultPort_default() { - final ManagedChannelImplBuilder builderNoPortProvider = new ManagedChannelImplBuilder( - DUMMY_TARGET, mockClientTransportFactoryBuilder, null); - assertEquals(GrpcUtil.DEFAULT_PORT_SSL, builderNoPortProvider.getDefaultPort()); + builder = new ManagedChannelImplBuilder(DUMMY_TARGET, + new UnsupportedClientTransportFactoryBuilder(), null); + assertEquals(GrpcUtil.DEFAULT_PORT_SSL, builder.getDefaultPort()); } /** Ensure getDefaultPort() delegates to the custom implementation. */ @Test public void getDefaultPort_custom() { - when(mockChannelBuilderDefaultPortProvider.getDefaultPort()).thenReturn(DUMMY_PORT); - assertEquals(DUMMY_PORT, builder.getDefaultPort()); + int customPort = 43; + ChannelBuilderDefaultPortProvider mockChannelBuilderDefaultPortProvider = mock( + ChannelBuilderDefaultPortProvider.class); + when(mockChannelBuilderDefaultPortProvider.getDefaultPort()).thenReturn(customPort); + + builder = new ManagedChannelImplBuilder(DUMMY_TARGET, + new UnsupportedClientTransportFactoryBuilder(), + mockChannelBuilderDefaultPortProvider); + assertEquals(customPort, builder.getDefaultPort()); verify(mockChannelBuilderDefaultPortProvider).getDefaultPort(); } /** Test FixedPortProvider(int port). */ @Test public void getDefaultPort_fixedPortProvider() { - final ManagedChannelImplBuilder builderFixedPortProvider = new ManagedChannelImplBuilder( - DUMMY_TARGET, - mockClientTransportFactoryBuilder, - new FixedPortProvider(DUMMY_PORT)); - assertEquals(DUMMY_PORT, builderFixedPortProvider.getDefaultPort()); + int fixedPort = 43; + builder = new ManagedChannelImplBuilder(DUMMY_TARGET, + new UnsupportedClientTransportFactoryBuilder(), new FixedPortProvider(fixedPort)); + assertEquals(fixedPort, builder.getDefaultPort()); + } + + @Test + public void executor_default() { + assertNotNull(builder.executorPool); + } + + @Test + public void executor_normal() { + Executor executor = mock(Executor.class); + assertEquals(builder, builder.executor(executor)); + assertEquals(executor, builder.executorPool.getObject()); + } + + @Test + public void executor_null() { + ObjectPool defaultValue = builder.executorPool; + builder.executor(mock(Executor.class)); + assertEquals(builder, builder.executor(null)); + assertEquals(defaultValue, builder.executorPool); + } + + @Test + public void directExecutor() { + assertEquals(builder, builder.directExecutor()); + assertEquals(MoreExecutors.directExecutor(), builder.executorPool.getObject()); + } + + @Test + public void offloadExecutor_normal() { + Executor executor = mock(Executor.class); + assertEquals(builder, builder.offloadExecutor(executor)); + assertEquals(executor, builder.offloadExecutorPool.getObject()); + } + + @Test + public void offloadExecutor_null() { + ObjectPool defaultValue = builder.offloadExecutorPool; + builder.offloadExecutor(mock(Executor.class)); + assertEquals(builder, builder.offloadExecutor(null)); + assertEquals(defaultValue, builder.offloadExecutorPool); + } + + @Test + public void nameResolverFactory_default() { + assertNotNull(builder.getNameResolverFactory()); + } + + @Test + @SuppressWarnings("deprecation") + public void nameResolverFactory_normal() { + NameResolver.Factory nameResolverFactory = mock(NameResolver.Factory.class); + assertEquals(builder, builder.nameResolverFactory(nameResolverFactory)); + assertEquals(nameResolverFactory, builder.getNameResolverFactory()); + } + + @Test + @SuppressWarnings("deprecation") + public void nameResolverFactory_null() { + NameResolver.Factory defaultValue = builder.getNameResolverFactory(); + builder.nameResolverFactory(mock(NameResolver.Factory.class)); + assertEquals(builder, builder.nameResolverFactory(null)); + assertEquals(defaultValue, builder.getNameResolverFactory()); + } + + @Test(expected = IllegalStateException.class) + @SuppressWarnings("deprecation") + public void nameResolverFactory_notAllowedWithDirectAddress() { + directAddressBuilder.nameResolverFactory(mock(NameResolver.Factory.class)); + } + + @Test + public void defaultLoadBalancingPolicy_default() { + assertEquals("pick_first", builder.defaultLbPolicy); + } + + @Test + public void defaultLoadBalancingPolicy_normal() { + assertEquals(builder, builder.defaultLoadBalancingPolicy("magic_balancer")); + assertEquals("magic_balancer", builder.defaultLbPolicy); + } + + @Test(expected = IllegalArgumentException.class) + public void defaultLoadBalancingPolicy_null() { + builder.defaultLoadBalancingPolicy(null); + } + + @Test(expected = IllegalStateException.class) + public void defaultLoadBalancingPolicy_notAllowedWithDirectAddress() { + directAddressBuilder.defaultLoadBalancingPolicy("magic_balancer"); + } + + @Test + public void fullStreamDecompression_default() { + assertFalse(builder.fullStreamDecompression); + } + + @Test + public void fullStreamDecompression_enabled() { + assertEquals(builder, builder.enableFullStreamDecompression()); + assertTrue(builder.fullStreamDecompression); + } + + @Test + public void decompressorRegistry_default() { + assertNotNull(builder.decompressorRegistry); + } + + @Test + public void decompressorRegistry_normal() { + DecompressorRegistry decompressorRegistry = DecompressorRegistry.emptyInstance(); + assertNotEquals(decompressorRegistry, builder.decompressorRegistry); + assertEquals(builder, builder.decompressorRegistry(decompressorRegistry)); + assertEquals(decompressorRegistry, builder.decompressorRegistry); + } + + @Test + public void decompressorRegistry_null() { + DecompressorRegistry defaultValue = builder.decompressorRegistry; + assertEquals(builder, builder.decompressorRegistry(DecompressorRegistry.emptyInstance())); + assertNotEquals(defaultValue, builder.decompressorRegistry); + builder.decompressorRegistry(null); + assertEquals(defaultValue, builder.decompressorRegistry); + } + + @Test + public void compressorRegistry_default() { + assertNotNull(builder.compressorRegistry); + } + + @Test + public void compressorRegistry_normal() { + CompressorRegistry compressorRegistry = CompressorRegistry.newEmptyInstance(); + assertNotEquals(compressorRegistry, builder.compressorRegistry); + assertEquals(builder, builder.compressorRegistry(compressorRegistry)); + assertEquals(compressorRegistry, builder.compressorRegistry); + } + + @Test + public void compressorRegistry_null() { + CompressorRegistry defaultValue = builder.compressorRegistry; + builder.compressorRegistry(CompressorRegistry.newEmptyInstance()); + assertNotEquals(defaultValue, builder.compressorRegistry); + assertEquals(builder, builder.compressorRegistry(null)); + assertEquals(defaultValue, builder.compressorRegistry); + } + + @Test + public void userAgent_default() { + assertNull(builder.userAgent); + } + + @Test + public void userAgent_normal() { + String userAgent = "user-agent/1"; + assertEquals(builder, builder.userAgent(userAgent)); + assertEquals(userAgent, builder.userAgent); + } + + @Test + public void userAgent_null() { + assertEquals(builder, builder.userAgent(null)); + assertNull(builder.userAgent); + + builder.userAgent("user-agent/1"); + builder.userAgent(null); + assertNull(builder.userAgent); + } + + @Test + public void authorityIsReadable_default() { + when(mockClientTransportFactory.getScheduledExecutorService()) + .thenReturn(clock.getScheduledExecutorService()); + when(mockClientTransportFactoryBuilder.buildClientTransportFactory()) + .thenReturn(mockClientTransportFactory); + + builder = new ManagedChannelImplBuilder(DUMMY_AUTHORITY_VALID, + mockClientTransportFactoryBuilder, new FixedPortProvider(DUMMY_PORT)); + ManagedChannel channel = grpcCleanupRule.register(builder.build()); + assertEquals(DUMMY_AUTHORITY_VALID, channel.authority()); + } + + @Test + public void authorityIsReadable_overrideAuthority() { + String overrideAuthority = "best-authority"; + when(mockClientTransportFactory.getScheduledExecutorService()) + .thenReturn(clock.getScheduledExecutorService()); + when(mockClientTransportFactoryBuilder.buildClientTransportFactory()) + .thenReturn(mockClientTransportFactory); + + builder = new ManagedChannelImplBuilder(DUMMY_TARGET, + mockClientTransportFactoryBuilder, new FixedPortProvider(DUMMY_PORT)) + .overrideAuthority(overrideAuthority); + ManagedChannel channel = grpcCleanupRule.register(builder.build()); + assertEquals(overrideAuthority, channel.authority()); + } + + @Test + public void overrideAuthority_default() { + assertNull(builder.getOverrideAuthority()); + } + + @Test + public void overrideAuthority_normal() { + String overrideAuthority = "best-authority"; + assertEquals(builder, builder.overrideAuthority(overrideAuthority)); + assertEquals(overrideAuthority, builder.getOverrideAuthority()); + } + + @Test(expected = NullPointerException.class) + public void overrideAuthority_null() { + builder.overrideAuthority(null); + } + + @Test(expected = IllegalArgumentException.class) + public void overrideAuthority_invalid() { + builder.overrideAuthority("not_allowed"); + } + + @Test + public void overrideAuthority_getNameResolverFactory() { + assertNull(builder.getOverrideAuthority()); + assertFalse(builder.getNameResolverFactory() instanceof OverrideAuthorityNameResolverFactory); + builder.overrideAuthority("google.com"); + assertTrue(builder.getNameResolverFactory() instanceof OverrideAuthorityNameResolverFactory); } @Test @@ -132,4 +400,202 @@ public void disableCheckAuthority_invalidAuthorityFailed() { builder.disableCheckAuthority().enableCheckAuthority(); builder.checkAuthority(DUMMY_AUTHORITY_INVALID); } + + @Test + public void makeTargetStringForDirectAddress_scopedIpv6() throws Exception { + InetSocketAddress address = new InetSocketAddress("0:0:0:0:0:0:0:0%0", 10005); + assertEquals("/0:0:0:0:0:0:0:0%0:10005", address.toString()); + String target = ManagedChannelImplBuilder.makeTargetStringForDirectAddress(address); + URI uri = new URI(target); + assertEquals("directaddress:////0:0:0:0:0:0:0:0%250:10005", target); + assertEquals(target, uri.toString()); + } + + @Test + public void getEffectiveInterceptors_default() { + builder.intercept(DUMMY_USER_INTERCEPTOR); + List effectiveInterceptors = builder.getEffectiveInterceptors(); + assertEquals(3, effectiveInterceptors.size()); + assertThat(effectiveInterceptors.get(0).getClass().getName()) + .isEqualTo("io.grpc.census.CensusTracingModule$TracingClientInterceptor"); + assertThat(effectiveInterceptors.get(1).getClass().getName()) + .isEqualTo("io.grpc.census.CensusStatsModule$StatsClientInterceptor"); + assertThat(effectiveInterceptors.get(2)).isSameInstanceAs(DUMMY_USER_INTERCEPTOR); + } + + @Test + public void getEffectiveInterceptors_disableStats() { + builder.intercept(DUMMY_USER_INTERCEPTOR); + builder.setStatsEnabled(false); + List effectiveInterceptors = builder.getEffectiveInterceptors(); + assertEquals(2, effectiveInterceptors.size()); + assertThat(effectiveInterceptors.get(0).getClass().getName()) + .isEqualTo("io.grpc.census.CensusTracingModule$TracingClientInterceptor"); + assertThat(effectiveInterceptors.get(1)).isSameInstanceAs(DUMMY_USER_INTERCEPTOR); + } + + @Test + public void getEffectiveInterceptors_disableTracing() { + builder.intercept(DUMMY_USER_INTERCEPTOR); + builder.setTracingEnabled(false); + List effectiveInterceptors = builder.getEffectiveInterceptors(); + assertEquals(2, effectiveInterceptors.size()); + assertThat(effectiveInterceptors.get(0).getClass().getName()) + .isEqualTo("io.grpc.census.CensusStatsModule$StatsClientInterceptor"); + assertThat(effectiveInterceptors.get(1)).isSameInstanceAs(DUMMY_USER_INTERCEPTOR); + } + + @Test + public void getEffectiveInterceptors_disableBoth() { + builder.intercept(DUMMY_USER_INTERCEPTOR); + builder.setStatsEnabled(false); + builder.setTracingEnabled(false); + List effectiveInterceptors = builder.getEffectiveInterceptors(); + assertThat(effectiveInterceptors).containsExactly(DUMMY_USER_INTERCEPTOR); + } + + @Test + public void idleTimeout() { + assertEquals(ManagedChannelImplBuilder.IDLE_MODE_DEFAULT_TIMEOUT_MILLIS, + builder.idleTimeoutMillis); + + builder.idleTimeout(Long.MAX_VALUE, TimeUnit.DAYS); + assertEquals(ManagedChannelImpl.IDLE_TIMEOUT_MILLIS_DISABLE, builder.idleTimeoutMillis); + + builder.idleTimeout(ManagedChannelImplBuilder.IDLE_MODE_MAX_TIMEOUT_DAYS, + TimeUnit.DAYS); + assertEquals(ManagedChannelImpl.IDLE_TIMEOUT_MILLIS_DISABLE, builder.idleTimeoutMillis); + + try { + builder.idleTimeout(0, TimeUnit.SECONDS); + fail("Should throw"); + } catch (IllegalArgumentException e) { + // expected + } + + builder.idleTimeout(1, TimeUnit.NANOSECONDS); + assertEquals(ManagedChannelImplBuilder.IDLE_MODE_MIN_TIMEOUT_MILLIS, + builder.idleTimeoutMillis); + + builder.idleTimeout(30, TimeUnit.SECONDS); + assertEquals(TimeUnit.SECONDS.toMillis(30), builder.idleTimeoutMillis); + } + + @Test + public void maxRetryAttempts() { + assertEquals(5, builder.maxRetryAttempts); + + builder.maxRetryAttempts(3); + assertEquals(3, builder.maxRetryAttempts); + } + + @Test + public void maxHedgedAttempts() { + assertEquals(5, builder.maxHedgedAttempts); + + builder.maxHedgedAttempts(3); + assertEquals(3, builder.maxHedgedAttempts); + } + + @Test + public void retryBufferSize() { + assertEquals(1L << 24, builder.retryBufferSize); + + builder.retryBufferSize(3456L); + assertEquals(3456L, builder.retryBufferSize); + } + + @Test + public void perRpcBufferLimit() { + assertEquals(1L << 20, builder.perRpcBufferLimit); + + builder.perRpcBufferLimit(3456L); + assertEquals(3456L, builder.perRpcBufferLimit); + } + + @Test + public void retryBufferSizeInvalidArg() { + thrown.expect(IllegalArgumentException.class); + builder.retryBufferSize(0L); + } + + @Test + public void perRpcBufferLimitInvalidArg() { + thrown.expect(IllegalArgumentException.class); + builder.perRpcBufferLimit(0L); + } + + @Test + public void disableRetry() { + builder.enableRetry(); + assertTrue(builder.retryEnabled); + + builder.disableRetry(); + assertFalse(builder.retryEnabled); + + builder.enableRetry(); + assertTrue(builder.retryEnabled); + + builder.disableRetry(); + assertFalse(builder.retryEnabled); + } + + @Test + public void defaultServiceConfig_nullKey() { + Map config = new HashMap<>(); + config.put(null, "val"); + + thrown.expect(IllegalArgumentException.class); + builder.defaultServiceConfig(config); + } + + @Test + public void defaultServiceConfig_intKey() { + Map subConfig = new HashMap<>(); + subConfig.put(3, "val"); + Map config = new HashMap<>(); + config.put("key", subConfig); + + thrown.expect(IllegalArgumentException.class); + builder.defaultServiceConfig(config); + } + + @Test + public void defaultServiceConfig_intValue() { + Map config = new HashMap<>(); + config.put("key", 3); + + thrown.expect(IllegalArgumentException.class); + builder.defaultServiceConfig(config); + } + + @Test + public void defaultServiceConfig_nested() { + Map config = new HashMap<>(); + List list1 = new ArrayList<>(); + list1.add(123D); + list1.add(null); + list1.add(true); + list1.add("str"); + Map map2 = new HashMap<>(); + map2.put("key2", false); + map2.put("key3", null); + map2.put("key4", Collections.singletonList("v4")); + map2.put("key4", 3.14D); + map2.put("key5", new HashMap()); + list1.add(map2); + config.put("key1", list1); + + builder.defaultServiceConfig(config); + + assertThat(builder.defaultServiceConfig).containsExactlyEntriesIn(config); + } + + @Test + public void disableNameResolverServiceConfig() { + assertThat(builder.lookUpServiceConfig).isTrue(); + + builder.disableServiceConfigLookUp(); + assertThat(builder.lookUpServiceConfig).isFalse(); + } } diff --git a/core/src/test/java/io/grpc/internal/ManagedChannelImplIdlenessTest.java b/core/src/test/java/io/grpc/internal/ManagedChannelImplIdlenessTest.java index c551d26449c..a2a2925c4f3 100644 --- a/core/src/test/java/io/grpc/internal/ManagedChannelImplIdlenessTest.java +++ b/core/src/test/java/io/grpc/internal/ManagedChannelImplIdlenessTest.java @@ -62,7 +62,7 @@ import io.grpc.Status; import io.grpc.StringMarshaller; import io.grpc.internal.FakeClock.ScheduledTask; -import io.grpc.internal.ManagedChannelImplBuilder.ClientTransportFactoryBuilder; +import io.grpc.internal.ManagedChannelImplBuilder.UnsupportedClientTransportFactoryBuilder; import io.grpc.internal.TestUtils.MockClientTransportInfo; import java.net.SocketAddress; import java.net.URI; @@ -161,12 +161,7 @@ public void setUp() { .thenReturn(timer.getScheduledExecutorService()); ManagedChannelImplBuilder builder = new ManagedChannelImplBuilder("fake://target", - new ClientTransportFactoryBuilder() { - @Override public ClientTransportFactory buildClientTransportFactory() { - throw new UnsupportedOperationException(); - } - }, - null); + new UnsupportedClientTransportFactoryBuilder(), null); builder .nameResolverFactory(mockNameResolverFactory) diff --git a/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java b/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java index 5a7933e4bf2..97208ae8631 100644 --- a/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java +++ b/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java @@ -109,6 +109,7 @@ import io.grpc.internal.ManagedChannelImpl.ScParser; import io.grpc.internal.ManagedChannelImplBuilder.ClientTransportFactoryBuilder; import io.grpc.internal.ManagedChannelImplBuilder.FixedPortProvider; +import io.grpc.internal.ManagedChannelImplBuilder.UnsupportedClientTransportFactoryBuilder; import io.grpc.internal.ServiceConfigUtil.PolicySelection; import io.grpc.internal.TestUtils.MockClientTransportInfo; import io.grpc.stub.ClientCalls; @@ -328,13 +329,8 @@ public void setUp() throws Exception { .thenReturn(balancerRpcExecutor.getScheduledExecutorService()); channelBuilder = new ManagedChannelImplBuilder(TARGET, - new ClientTransportFactoryBuilder() { - @Override - public ClientTransportFactory buildClientTransportFactory() { - throw new UnsupportedOperationException(); - } - }, - new FixedPortProvider(DEFAULT_PORT)); + new UnsupportedClientTransportFactoryBuilder(), new FixedPortProvider(DEFAULT_PORT)); + channelBuilder .nameResolverFactory(new FakeNameResolverFactory.Builder(expectedUri).build()) .defaultLoadBalancingPolicy(MOCK_POLICY_NAME) diff --git a/core/src/test/java/io/grpc/internal/ServiceConfigErrorHandlingTest.java b/core/src/test/java/io/grpc/internal/ServiceConfigErrorHandlingTest.java index 49f094a6adb..16c6f3bf302 100644 --- a/core/src/test/java/io/grpc/internal/ServiceConfigErrorHandlingTest.java +++ b/core/src/test/java/io/grpc/internal/ServiceConfigErrorHandlingTest.java @@ -46,8 +46,8 @@ import io.grpc.NameResolver; import io.grpc.NameResolver.ConfigOrError; import io.grpc.Status; -import io.grpc.internal.ManagedChannelImplBuilder.ClientTransportFactoryBuilder; import io.grpc.internal.ManagedChannelImplBuilder.FixedPortProvider; +import io.grpc.internal.ManagedChannelImplBuilder.UnsupportedClientTransportFactoryBuilder; import java.net.SocketAddress; import java.net.URI; import java.util.ArrayList; @@ -200,13 +200,7 @@ public void setUp() throws Exception { when(executorPool.getObject()).thenReturn(executor.getScheduledExecutorService()); channelBuilder = new ManagedChannelImplBuilder(TARGET, - new ClientTransportFactoryBuilder() { - @Override - public ClientTransportFactory buildClientTransportFactory() { - throw new UnsupportedOperationException(); - } - }, - new FixedPortProvider(DEFAULT_PORT)); + new UnsupportedClientTransportFactoryBuilder(), new FixedPortProvider(DEFAULT_PORT)); channelBuilder .nameResolverFactory(new FakeNameResolverFactory.Builder(expectedUri).build()) From b434df25cd80522913b27ba9960ba352453cf84e Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Tue, 22 Sep 2020 16:08:41 -0700 Subject: [PATCH 59/86] xds: generate EDS LB config with hardcoded locality picking policy (#7443) --- .../java/io/grpc/xds/CdsLoadBalancer.java | 14 ++++--- .../io/grpc/xds/EdsLoadBalancerProvider.java | 17 -------- .../java/io/grpc/xds/CdsLoadBalancerTest.java | 7 ++++ .../java/io/grpc/xds/EdsLoadBalancerTest.java | 41 ++++++++++++------- 4 files changed, 41 insertions(+), 38 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java index 5dfe6d5444d..3d1cb31bcc2 100644 --- a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java @@ -20,9 +20,9 @@ import static com.google.common.base.Preconditions.checkNotNull; import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; import static io.grpc.xds.XdsLbPolicies.EDS_POLICY_NAME; +import static io.grpc.xds.XdsLbPolicies.WEIGHTED_TARGET_POLICY_NAME; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableMap; import io.grpc.EquivalentAddressGroup; import io.grpc.InternalLogId; import io.grpc.LoadBalancer; @@ -198,18 +198,20 @@ public void onClusterChanged(ClusterUpdate newUpdate) { xdsClient, newUpdate.getClusterName(), newUpdate.getEdsServiceName(), newUpdate.getLbPolicy(), newUpdate.getLrsServerName() != null); } + // FIXME(chengyuanzhang): handle error correctly to avoid being unnecessarily fragile. checkArgument( newUpdate.getLbPolicy().equals("round_robin"), "can only support round_robin policy"); - - LoadBalancerProvider lbProvider = lbRegistry.getProvider(newUpdate.getLbPolicy()); - Object lbConfig = - lbProvider.parseLoadBalancingPolicyConfig(ImmutableMap.of()).getConfig(); + LoadBalancerProvider endpointPickingPolicyProvider = + lbRegistry.getProvider(newUpdate.getLbPolicy()); + LoadBalancerProvider localityPickingPolicyProvider = + lbRegistry.getProvider(WEIGHTED_TARGET_POLICY_NAME); // hardcode to weighted-target final EdsConfig edsConfig = new EdsConfig( /* clusterName = */ newUpdate.getClusterName(), /* edsServiceName = */ newUpdate.getEdsServiceName(), /* lrsServerName = */ newUpdate.getLrsServerName(), - new PolicySelection(lbProvider, ImmutableMap.of(), lbConfig)); + new PolicySelection(localityPickingPolicyProvider, null, null /* by EDS policy */), + new PolicySelection(endpointPickingPolicyProvider, null, null)); if (isXdsSecurityEnabled()) { updateSslContextProviderSupplier(newUpdate.getUpstreamTlsContext()); } diff --git a/xds/src/main/java/io/grpc/xds/EdsLoadBalancerProvider.java b/xds/src/main/java/io/grpc/xds/EdsLoadBalancerProvider.java index fb44cecf054..57830954036 100644 --- a/xds/src/main/java/io/grpc/xds/EdsLoadBalancerProvider.java +++ b/xds/src/main/java/io/grpc/xds/EdsLoadBalancerProvider.java @@ -17,7 +17,6 @@ package io.grpc.xds; import static com.google.common.base.Preconditions.checkNotNull; -import static io.grpc.xds.XdsLbPolicies.WEIGHTED_TARGET_POLICY_NAME; import com.google.common.base.MoreObjects; import com.google.common.base.Objects; @@ -25,7 +24,6 @@ import io.grpc.LoadBalancer; import io.grpc.LoadBalancer.Helper; import io.grpc.LoadBalancerProvider; -import io.grpc.LoadBalancerRegistry; import io.grpc.NameResolver.ConfigOrError; import io.grpc.internal.ServiceConfigUtil.PolicySelection; import java.util.Map; @@ -74,21 +72,6 @@ static final class EdsConfig { final PolicySelection localityPickingPolicy; final PolicySelection endpointPickingPolicy; - // TODO(chengyuanzhang): delete me. - EdsConfig( - String clusterName, - @Nullable String edsServiceName, - @Nullable String lrsServerName, - PolicySelection endpointPickingPolicy) { - this.clusterName = checkNotNull(clusterName, "clusterName"); - this.edsServiceName = edsServiceName; - this.lrsServerName = lrsServerName; - this.endpointPickingPolicy = checkNotNull(endpointPickingPolicy, "endpointPickingPolicy"); - LoadBalancerProvider provider = - LoadBalancerRegistry.getDefaultRegistry().getProvider(WEIGHTED_TARGET_POLICY_NAME); - localityPickingPolicy = new PolicySelection(provider, null, null); - } - EdsConfig( String clusterName, @Nullable String edsServiceName, diff --git a/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java index 4c0a1d4b324..d7570725e0b 100644 --- a/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java @@ -89,6 +89,7 @@ public void setUp() { MockitoAnnotations.initMocks(this); LoadBalancerRegistry registry = new LoadBalancerRegistry(); + registry.register(new FakeLoadBalancerProvider(XdsLbPolicies.WEIGHTED_TARGET_POLICY_NAME)); registry.register(new FakeLoadBalancerProvider(XdsLbPolicies.EDS_POLICY_NAME)); registry.register(new FakeLoadBalancerProvider("round_robin")); ObjectPool xdsClientPool = new ObjectPool() { @@ -135,6 +136,8 @@ public void receiveFirstClusterResourceInfo() { assertThat(edsConfig.clusterName).isEqualTo(CLUSTER); assertThat(edsConfig.edsServiceName).isNull(); assertThat(edsConfig.lrsServerName).isNull(); + assertThat(edsConfig.localityPickingPolicy.getProvider().getPolicyName()) + .isEqualTo(XdsLbPolicies.WEIGHTED_TARGET_POLICY_NAME); // hardcoded to weighted-target assertThat(edsConfig.endpointPickingPolicy.getProvider().getPolicyName()) .isEqualTo("round_robin"); } @@ -174,6 +177,8 @@ public void clusterResourceUpdated() { assertThat(edsConfig.clusterName).isEqualTo(CLUSTER); assertThat(edsConfig.edsServiceName).isNull(); assertThat(edsConfig.lrsServerName).isNull(); + assertThat(edsConfig.localityPickingPolicy.getProvider().getPolicyName()) + .isEqualTo(XdsLbPolicies.WEIGHTED_TARGET_POLICY_NAME); // hardcoded to weighted-target assertThat(edsConfig.endpointPickingPolicy.getProvider().getPolicyName()) .isEqualTo("round_robin"); @@ -185,6 +190,8 @@ public void clusterResourceUpdated() { assertThat(edsConfig.clusterName).isEqualTo(CLUSTER); assertThat(edsConfig.edsServiceName).isEqualTo(edsService); assertThat(edsConfig.lrsServerName).isEqualTo(loadReportServer); + assertThat(edsConfig.localityPickingPolicy.getProvider().getPolicyName()) + .isEqualTo(XdsLbPolicies.WEIGHTED_TARGET_POLICY_NAME); // hardcoded to weighted-target assertThat(edsConfig.endpointPickingPolicy.getProvider().getPolicyName()) .isEqualTo("round_robin"); } diff --git a/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java index 192201e7a94..be78db727d6 100644 --- a/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java @@ -143,6 +143,8 @@ XdsChannel createChannel(List servers) { } }; + private final PolicySelection fakeLocalityPickingPolicy = + new PolicySelection(mock(LoadBalancerProvider.class), null, null); private final PolicySelection fakeEndpointPickingPolicy = new PolicySelection(mock(LoadBalancerProvider.class), null, new Object()); @@ -272,7 +274,7 @@ public void tearDown() { @Test public void handleNameResolutionErrorBeforeAndAfterEdsWorkding() { - deliverResolvedAddresses(null, null, fakeEndpointPickingPolicy); + deliverResolvedAddresses(null, null, fakeLocalityPickingPolicy, fakeEndpointPickingPolicy); // handleResolutionError() before receiving any endpoint update. edsLb.handleNameResolutionError(Status.DATA_LOSS.withDescription("fake status")); @@ -300,7 +302,8 @@ public void handleNameResolutionErrorBeforeAndAfterEdsWorkding() { public void handleEdsServiceNameChange() { assertThat(childHelpers).isEmpty(); - deliverResolvedAddresses("edsServiceName1", null, fakeEndpointPickingPolicy); + deliverResolvedAddresses("edsServiceName1", null, fakeLocalityPickingPolicy, + fakeEndpointPickingPolicy); ClusterLoadAssignment clusterLoadAssignment = buildClusterLoadAssignmentV2("edsServiceName1", ImmutableList.of( @@ -320,7 +323,8 @@ public void handleEdsServiceNameChange() { assertLatestConnectivityState(CONNECTING); // Change edsServicename to edsServiceName2. - deliverResolvedAddresses("edsServiceName2", null, fakeEndpointPickingPolicy); + deliverResolvedAddresses("edsServiceName2", null, fakeLocalityPickingPolicy, + fakeEndpointPickingPolicy); // The old balancer was not READY, so it will be shutdown immediately. verify(childBalancer1).shutdown(); @@ -350,7 +354,8 @@ public PickResult pickSubchannel(PickSubchannelArgs args) { assertLatestSubchannelPicker(subchannel2); // Change edsServiceName to edsServiceName3. - deliverResolvedAddresses("edsServiceName3", null, fakeEndpointPickingPolicy); + deliverResolvedAddresses("edsServiceName3", null, fakeLocalityPickingPolicy, + fakeEndpointPickingPolicy); clusterLoadAssignment = buildClusterLoadAssignmentV2("edsServiceName3", ImmutableList.of( @@ -376,7 +381,8 @@ public PickResult pickSubchannel(PickSubchannelArgs args) { assertLatestConnectivityState(CONNECTING); // Change edsServiceName to edsServiceName4. - deliverResolvedAddresses("edsServiceName4", null, fakeEndpointPickingPolicy); + deliverResolvedAddresses("edsServiceName4", null, fakeLocalityPickingPolicy, + fakeEndpointPickingPolicy); verify(childBalancer3).shutdown(); clusterLoadAssignment = @@ -404,7 +410,8 @@ public PickResult pickSubchannel(PickSubchannelArgs args) { assertLatestSubchannelPicker(subchannel4); // Change edsServiceName to edsServiceName5. - deliverResolvedAddresses("edsServiceName5", null, fakeEndpointPickingPolicy); + deliverResolvedAddresses("edsServiceName5", null, fakeLocalityPickingPolicy, + fakeEndpointPickingPolicy); clusterLoadAssignment = buildClusterLoadAssignmentV2("edsServiceName5", ImmutableList.of( @@ -439,7 +446,7 @@ public PickResult pickSubchannel(PickSubchannelArgs args) { @Test public void edsResourceUpdate_allDrop() { - deliverResolvedAddresses(null, null, fakeEndpointPickingPolicy); + deliverResolvedAddresses(null, null, fakeLocalityPickingPolicy, fakeEndpointPickingPolicy); ClusterLoadAssignment clusterLoadAssignment = buildClusterLoadAssignmentV2( CLUSTER_NAME, @@ -488,7 +495,7 @@ public PickResult pickSubchannel(PickSubchannelArgs args) { @Test public void edsResourceUpdate_localityAssignmentChange() { - deliverResolvedAddresses(null, null, fakeEndpointPickingPolicy); + deliverResolvedAddresses(null, null, fakeLocalityPickingPolicy, fakeEndpointPickingPolicy); LbEndpoint endpoint11 = buildLbEndpointV2("addr11.example.com", 8011, HEALTHY, 11); LbEndpoint endpoint12 = buildLbEndpointV2("addr12.example.com", 8012, HEALTHY, 12); @@ -575,7 +582,8 @@ LocalityStore newLocalityStore( edsLb = new EdsLoadBalancer(helper, lbRegistry, localityStoreFactory, bootstrapper, channelFactory); - deliverResolvedAddresses("edsServiceName1", null, fakeEndpointPickingPolicy); + deliverResolvedAddresses("edsServiceName1", null, fakeLocalityPickingPolicy, + fakeEndpointPickingPolicy); assertThat(localityStores).hasSize(1); LocalityStore localityStore = localityStores.peekLast(); @@ -612,7 +620,8 @@ LocalityStore newLocalityStore( verify(localityStore).updateLocalityStore(endpointUpdate.getLocalityLbEndpointsMap()); // Change cluster name. - deliverResolvedAddresses("edsServiceName2", null, fakeEndpointPickingPolicy); + deliverResolvedAddresses("edsServiceName2", null, fakeLocalityPickingPolicy, + fakeEndpointPickingPolicy); assertThat(localityStores).hasSize(2); localityStore = localityStores.peekLast(); @@ -635,7 +644,7 @@ LocalityStore newLocalityStore( @Test public void edsResourceNotExist() { - deliverResolvedAddresses(null, null, fakeEndpointPickingPolicy); + deliverResolvedAddresses(null, null, fakeLocalityPickingPolicy, fakeEndpointPickingPolicy); // Forwarding 20 seconds so that the xds client will deem EDS resource not available. fakeClock.forwardTime(20, TimeUnit.SECONDS); @@ -649,7 +658,7 @@ public void edsResourceNotExist() { @Test public void edsResourceRemoved() { - deliverResolvedAddresses(null, null, fakeEndpointPickingPolicy); + deliverResolvedAddresses(null, null, fakeLocalityPickingPolicy, fakeEndpointPickingPolicy); ClusterLoadAssignment clusterLoadAssignment = buildClusterLoadAssignmentV2(CLUSTER_NAME, ImmutableList.of( @@ -695,7 +704,7 @@ public PickResult pickSubchannel(PickSubchannelArgs args) { @Test public void transientError_noPreviousEndpointUpdateReceived() { - deliverResolvedAddresses(null, null, fakeEndpointPickingPolicy); + deliverResolvedAddresses(null, null, fakeLocalityPickingPolicy, fakeEndpointPickingPolicy); // Forwarding 20 seconds so that the xds client will deem EDS resource not available. fakeClock.forwardTime(20, TimeUnit.SECONDS); @@ -704,7 +713,7 @@ public void transientError_noPreviousEndpointUpdateReceived() { @Test public void transientError_withPreviousEndpointUpdateReceived() { - deliverResolvedAddresses(null, null, fakeEndpointPickingPolicy); + deliverResolvedAddresses(null, null, fakeLocalityPickingPolicy, fakeEndpointPickingPolicy); // Endpoint update received. ClusterLoadAssignment clusterLoadAssignment = buildClusterLoadAssignmentV2(CLUSTER_NAME, @@ -751,9 +760,11 @@ private static EndpointUpdate getEndpointUpdateFromClusterAssignmentV2( private void deliverResolvedAddresses( @Nullable String edsServiceName, @Nullable String lrsServerName, + PolicySelection locaityPickingPolicy, PolicySelection endpointPickingPolicy) { EdsConfig config = - new EdsConfig(CLUSTER_NAME, edsServiceName, lrsServerName, endpointPickingPolicy); + new EdsConfig(CLUSTER_NAME, edsServiceName, lrsServerName, locaityPickingPolicy, + endpointPickingPolicy); ResolvedAddresses.Builder resolvedAddressBuilder = ResolvedAddresses.newBuilder() .setAddresses(ImmutableList.of()) .setLoadBalancingPolicyConfig(config); From bc8c758a3c2e3f355e81a293f3dadc965c72064a Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Tue, 22 Sep 2020 17:28:45 -0700 Subject: [PATCH 60/86] xds: log raw response messages in sync context (#7441) --- .../java/io/grpc/xds/LoadReportClient.java | 119 +++++++----- .../main/java/io/grpc/xds/XdsClientImpl.java | 182 +++++++++--------- 2 files changed, 164 insertions(+), 137 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/LoadReportClient.java b/xds/src/main/java/io/grpc/xds/LoadReportClient.java index 119ee4dfdcc..48d7b0a5cb1 100644 --- a/xds/src/main/java/io/grpc/xds/LoadReportClient.java +++ b/xds/src/main/java/io/grpc/xds/LoadReportClient.java @@ -185,49 +185,36 @@ private abstract class LrsStream { abstract void sendError(Exception error); - final void handleResponse(final LoadStatsResponseData response) { - syncContext.execute(new Runnable() { - @Override - public void run() { - if (closed) { - return; - } - if (!initialResponseReceived) { - logger.log(XdsLogLevel.DEBUG, "Initial LRS response received"); - initialResponseReceived = true; - } - reportAllClusters = response.getSendAllClusters(); - if (reportAllClusters) { - logger.log(XdsLogLevel.INFO, "Report loads for all clusters"); - } else { - logger.log(XdsLogLevel.INFO, "Report loads for clusters: ", response.getClustersList()); - clusterNames = response.getClustersList(); - } - long interval = response.getLoadReportingIntervalNanos(); - logger.log(XdsLogLevel.INFO, "Update load reporting interval to {0} ns", interval); - loadReportIntervalNano = interval; - scheduleNextLoadReport(); - } - }); + // Must run in syncContext. + final void handleResponse(LoadStatsResponseData response) { + if (closed) { + return; + } + if (!initialResponseReceived) { + logger.log(XdsLogLevel.DEBUG, "Initial LRS response received"); + initialResponseReceived = true; + } + reportAllClusters = response.getSendAllClusters(); + if (reportAllClusters) { + logger.log(XdsLogLevel.INFO, "Report loads for all clusters"); + } else { + logger.log(XdsLogLevel.INFO, "Report loads for clusters: ", response.getClustersList()); + clusterNames = response.getClustersList(); + } + long interval = response.getLoadReportingIntervalNanos(); + logger.log(XdsLogLevel.INFO, "Update load reporting interval to {0} ns", interval); + loadReportIntervalNano = interval; + scheduleNextLoadReport(); } - final void handleRpcError(final Throwable t) { - syncContext.execute(new Runnable() { - @Override - public void run() { - handleStreamClosed(Status.fromThrowable(t)); - } - }); + // Must run in syncContext. + final void handleRpcError(Throwable t) { + handleStreamClosed(Status.fromThrowable(t)); } - final void handleRpcComplete() { - syncContext.execute(new Runnable() { - @Override - public void run() { - handleStreamClosed( - Status.UNAVAILABLE.withDescription("Closed by server")); - } - }); + // Must run in syncContext. + final void handleRpcCompleted() { + handleStreamClosed(Status.UNAVAILABLE.withDescription("Closed by server")); } private void sendLoadReport() { @@ -324,19 +311,34 @@ void start() { new StreamObserver() { @Override public void onNext( - io.envoyproxy.envoy.service.load_stats.v2.LoadStatsResponse response) { - logger.log(XdsLogLevel.DEBUG, "Received LRS response:\n{0}", response); - handleResponse(LoadStatsResponseData.fromEnvoyProtoV2(response)); + final io.envoyproxy.envoy.service.load_stats.v2.LoadStatsResponse response) { + syncContext.execute(new Runnable() { + @Override + public void run() { + logger.log(XdsLogLevel.DEBUG, "Received LoadStatsResponse:\n{0}", response); + handleResponse(LoadStatsResponseData.fromEnvoyProtoV2(response)); + } + }); } @Override - public void onError(Throwable t) { - handleRpcError(t); + public void onError(final Throwable t) { + syncContext.execute(new Runnable() { + @Override + public void run() { + handleRpcError(t); + } + }); } @Override public void onCompleted() { - handleRpcComplete(); + syncContext.execute(new Runnable() { + @Override + public void run() { + handleRpcCompleted(); + } + }); } }; io.envoyproxy.envoy.service.load_stats.v2.LoadReportingServiceGrpc.LoadReportingServiceStub @@ -369,19 +371,34 @@ void start() { StreamObserver lrsResponseReaderV3 = new StreamObserver() { @Override - public void onNext(LoadStatsResponse response) { - logger.log(XdsLogLevel.DEBUG, "Received LRS response:\n{0}", response); - handleResponse(LoadStatsResponseData.fromEnvoyProtoV3(response)); + public void onNext(final LoadStatsResponse response) { + syncContext.execute(new Runnable() { + @Override + public void run() { + logger.log(XdsLogLevel.DEBUG, "Received LRS response:\n{0}", response); + handleResponse(LoadStatsResponseData.fromEnvoyProtoV3(response)); + } + }); } @Override - public void onError(Throwable t) { - handleRpcError(t); + public void onError(final Throwable t) { + syncContext.execute(new Runnable() { + @Override + public void run() { + handleRpcError(t); + } + }); } @Override public void onCompleted() { - handleRpcComplete(); + syncContext.execute(new Runnable() { + @Override + public void run() { + handleRpcCompleted(); + } + }); } }; LoadReportingServiceStub stubV3 = diff --git a/xds/src/main/java/io/grpc/xds/XdsClientImpl.java b/xds/src/main/java/io/grpc/xds/XdsClientImpl.java index b427cdd7c90..ba43da1796d 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientImpl.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientImpl.java @@ -1455,68 +1455,54 @@ private abstract class AbstractAdsStream { abstract void sendError(Exception error); - final void onDiscoveryResponse(final DiscoveryResponseData response) { - syncContext.execute( - new Runnable() { - @Override - public void run() { - if (closed) { - return; - } - responseReceived = true; - String respNonce = response.getNonce(); - // Nonce in each response is echoed back in the following ACK/NACK request. It is - // used for management server to identify which response the client is ACKing/NACking. - // To avoid confusion, client-initiated requests will always use the nonce in - // most recently received responses of each resource type. - ResourceType resourceType = response.getResourceType(); - switch (resourceType) { - case LDS: - ldsRespNonce = respNonce; - handleLdsResponse(response); - break; - case RDS: - rdsRespNonce = respNonce; - handleRdsResponse(response); - break; - case CDS: - cdsRespNonce = respNonce; - handleCdsResponse(response); - break; - case EDS: - edsRespNonce = respNonce; - handleEdsResponse(response); - break; - case UNKNOWN: - logger.log( - XdsLogLevel.WARNING, - "Received an unknown type of DiscoveryResponse\n{0}", - respNonce); - break; - default: - throw new AssertionError("Missing case in enum switch: " + resourceType); - } - } - }); + // Must run in syncContext. + final void handleResponse(DiscoveryResponseData response) { + if (closed) { + return; + } + responseReceived = true; + String respNonce = response.getNonce(); + // Nonce in each response is echoed back in the following ACK/NACK request. It is + // used for management server to identify which response the client is ACKing/NACking. + // To avoid confusion, client-initiated requests will always use the nonce in + // most recently received responses of each resource type. + ResourceType resourceType = response.getResourceType(); + switch (resourceType) { + case LDS: + ldsRespNonce = respNonce; + handleLdsResponse(response); + break; + case RDS: + rdsRespNonce = respNonce; + handleRdsResponse(response); + break; + case CDS: + cdsRespNonce = respNonce; + handleCdsResponse(response); + break; + case EDS: + edsRespNonce = respNonce; + handleEdsResponse(response); + break; + case UNKNOWN: + logger.log( + XdsLogLevel.WARNING, + "Received an unknown type of DiscoveryResponse\n{0}", + respNonce); + break; + default: + throw new AssertionError("Missing case in enum switch: " + resourceType); + } } - final void onError(final Throwable t) { - syncContext.execute(new Runnable() { - @Override - public void run() { - handleStreamClosed(Status.fromThrowable(t)); - } - }); + // Must run in syncContext. + final void handleRpcError(Throwable t) { + handleStreamClosed(Status.fromThrowable(t)); } - final void onCompleted() { - syncContext.execute(new Runnable() { - @Override - public void run() { - handleStreamClosed( - Status.UNAVAILABLE.withDescription("Closed by server")); - } - }); + // Must run in syncContext. + final void handleRpcCompleted() { + handleStreamClosed(Status.UNAVAILABLE.withDescription("Closed by server")); } private void handleStreamClosed(Status error) { @@ -1733,27 +1719,40 @@ void start() { StreamObserver responseReaderV2 = new StreamObserver() { @Override - public void onNext(io.envoyproxy.envoy.api.v2.DiscoveryResponse response) { - DiscoveryResponseData responseData = - DiscoveryResponseData.fromEnvoyProtoV2(response); - if (logger.isLoggable(XdsLogLevel.DEBUG)) { - logger.log( - XdsLogLevel.DEBUG, - "Received {0} response:\n{1}", - responseData.getResourceType(), - respPrinter.print(response)); - } - onDiscoveryResponse(responseData); + public void onNext(final io.envoyproxy.envoy.api.v2.DiscoveryResponse response) { + syncContext.execute(new Runnable() { + @Override + public void run() { + if (logger.isLoggable(XdsLogLevel.DEBUG)) { + logger.log(XdsLogLevel.DEBUG, "Received {0} response:\n{1}", + ResourceType.fromTypeUrl(response.getTypeUrl()), + respPrinter.print(response)); + } + DiscoveryResponseData responseData = + DiscoveryResponseData.fromEnvoyProtoV2(response); + handleResponse(responseData); + } + }); } @Override - public void onError(Throwable t) { - AdsStreamV2.this.onError(t); + public void onError(final Throwable t) { + syncContext.execute(new Runnable() { + @Override + public void run() { + handleRpcError(t); + } + }); } @Override public void onCompleted() { - AdsStreamV2.this.onCompleted(); + syncContext.execute(new Runnable() { + @Override + public void run() { + handleRpcCompleted(); + } + }); } }; requestWriterV2 = stubV2.withWaitForReady().streamAggregatedResources(responseReaderV2); @@ -1787,27 +1786,38 @@ private final class AdsStream extends AbstractAdsStream { void start() { StreamObserver responseReader = new StreamObserver() { @Override - public void onNext(DiscoveryResponse response) { - DiscoveryResponseData responseData = - DiscoveryResponseData.fromEnvoyProto(response); - if (logger.isLoggable(XdsLogLevel.DEBUG)) { - logger.log( - XdsLogLevel.DEBUG, - "Received {0} response:\n{1}", - responseData.getResourceType(), - respPrinter.print(response)); - } - onDiscoveryResponse(responseData); + public void onNext(final DiscoveryResponse response) { + syncContext.execute(new Runnable() { + @Override + public void run() { + if (logger.isLoggable(XdsLogLevel.DEBUG)) { + logger.log(XdsLogLevel.DEBUG, "Received {0} response:\n{1}", + ResourceType.fromTypeUrl(response.getTypeUrl()), respPrinter.print(response)); + } + DiscoveryResponseData responseData = DiscoveryResponseData.fromEnvoyProto(response); + handleResponse(responseData); + } + }); } @Override - public void onError(Throwable t) { - AdsStream.this.onError(t); + public void onError(final Throwable t) { + syncContext.execute(new Runnable() { + @Override + public void run() { + handleRpcError(t); + } + }); } @Override public void onCompleted() { - AdsStream.this.onCompleted(); + syncContext.execute(new Runnable() { + @Override + public void run() { + handleRpcCompleted(); + } + }); } }; requestWriter = stub.withWaitForReady().streamAggregatedResources(responseReader); From d5668b9ee151cf156b304fdda607b7c0dee1df9d Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Tue, 22 Sep 2020 18:30:54 -0700 Subject: [PATCH 61/86] core,xds: remove deprecated rawConfig field from PolicySelection --- .../AutoConfiguredLoadBalancerFactory.java | 5 ++- .../io/grpc/internal/ManagedChannelImpl.java | 10 ++---- .../io/grpc/internal/ServiceConfigUtil.java | 16 +++------ ...AutoConfiguredLoadBalancerFactoryTest.java | 36 +++++++++---------- .../grpc/internal/ManagedChannelImplTest.java | 14 +++----- .../java/io/grpc/xds/CdsLoadBalancer.java | 4 +-- .../java/io/grpc/xds/EdsLoadBalancer2.java | 4 +-- ...lusterManagerLoadBalancerProviderTest.java | 5 ++- .../xds/ClusterManagerLoadBalancerTest.java | 2 +- .../io/grpc/xds/EdsLoadBalancer2Test.java | 10 +++--- .../java/io/grpc/xds/EdsLoadBalancerTest.java | 4 +-- .../java/io/grpc/xds/LrsLoadBalancerTest.java | 2 +- .../xds/PriorityLoadBalancerProviderTest.java | 4 +-- .../io/grpc/xds/PriorityLoadBalancerTest.java | 20 +++++------ ...eightedTargetLoadBalancerProviderTest.java | 5 ++- .../xds/WeightedTargetLoadBalancerTest.java | 16 ++++----- .../XdsRoutingLoadBalancerProviderTest.java | 5 ++- .../grpc/xds/XdsRoutingLoadBalancerTest.java | 2 +- 18 files changed, 72 insertions(+), 92 deletions(-) diff --git a/core/src/main/java/io/grpc/internal/AutoConfiguredLoadBalancerFactory.java b/core/src/main/java/io/grpc/internal/AutoConfiguredLoadBalancerFactory.java index 24740d7bb36..037d49cb72f 100644 --- a/core/src/main/java/io/grpc/internal/AutoConfiguredLoadBalancerFactory.java +++ b/core/src/main/java/io/grpc/internal/AutoConfiguredLoadBalancerFactory.java @@ -21,7 +21,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; import io.grpc.Attributes; -import io.grpc.ChannelLogger; import io.grpc.ChannelLogger.ChannelLogLevel; import io.grpc.ConnectivityState; import io.grpc.ConnectivityStateInfo; @@ -126,7 +125,7 @@ Status tryHandleResolvedAddresses(ResolvedAddresses resolvedAddresses) { return Status.OK; } policySelection = - new PolicySelection(defaultProvider, /* rawConfig= */ null, /* config= */ null); + new PolicySelection(defaultProvider, /* config= */ null); } if (delegateProvider == null @@ -227,7 +226,7 @@ private LoadBalancerProvider getProviderOrThrow(String policy, String choiceReas * @return the parsed {@link PolicySelection}, or {@code null} if no selection could be made. */ @Nullable - ConfigOrError parseLoadBalancerPolicy(Map serviceConfig, ChannelLogger channelLogger) { + ConfigOrError parseLoadBalancerPolicy(Map serviceConfig) { try { List loadBalancerConfigs = null; if (serviceConfig != null) { diff --git a/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java b/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java index d2bc87cdde3..117e02ad531 100644 --- a/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java +++ b/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java @@ -612,8 +612,7 @@ ClientStream newSubstream(ClientStreamTracer.Factory tracerFactory, Metadata new retryEnabled, builder.maxRetryAttempts, builder.maxHedgedAttempts, - loadBalancerFactory, - channelLogger); + loadBalancerFactory); this.nameResolverArgs = NameResolver.Args.newBuilder() .setDefaultPort(builder.getDefaultPort()) @@ -2146,20 +2145,17 @@ static final class ScParser extends NameResolver.ServiceConfigParser { private final int maxRetryAttemptsLimit; private final int maxHedgedAttemptsLimit; private final AutoConfiguredLoadBalancerFactory autoLoadBalancerFactory; - private final ChannelLogger channelLogger; ScParser( boolean retryEnabled, int maxRetryAttemptsLimit, int maxHedgedAttemptsLimit, - AutoConfiguredLoadBalancerFactory autoLoadBalancerFactory, - ChannelLogger channelLogger) { + AutoConfiguredLoadBalancerFactory autoLoadBalancerFactory) { this.retryEnabled = retryEnabled; this.maxRetryAttemptsLimit = maxRetryAttemptsLimit; this.maxHedgedAttemptsLimit = maxHedgedAttemptsLimit; this.autoLoadBalancerFactory = checkNotNull(autoLoadBalancerFactory, "autoLoadBalancerFactory"); - this.channelLogger = checkNotNull(channelLogger, "channelLogger"); } @Override @@ -2167,7 +2163,7 @@ public ConfigOrError parseServiceConfig(Map rawServiceConfig) { try { Object loadBalancingPolicySelection; ConfigOrError choiceFromLoadBalancer = - autoLoadBalancerFactory.parseLoadBalancerPolicy(rawServiceConfig, channelLogger); + autoLoadBalancerFactory.parseLoadBalancerPolicy(rawServiceConfig); if (choiceFromLoadBalancer == null) { loadBalancingPolicySelection = null; } else if (choiceFromLoadBalancer.getError() != null) { diff --git a/core/src/main/java/io/grpc/internal/ServiceConfigUtil.java b/core/src/main/java/io/grpc/internal/ServiceConfigUtil.java index c8ff0d0b76e..e6df6240d19 100644 --- a/core/src/main/java/io/grpc/internal/ServiceConfigUtil.java +++ b/core/src/main/java/io/grpc/internal/ServiceConfigUtil.java @@ -352,8 +352,8 @@ public static ConfigOrError selectLbPolicyFromList( if (parsedLbPolicyConfig.getError() != null) { return parsedLbPolicyConfig; } - return ConfigOrError.fromConfig(new PolicySelection( - provider, lbConfig.rawConfigValue, parsedLbPolicyConfig.getConfig())); + return ConfigOrError.fromConfig( + new PolicySelection(provider, parsedLbPolicyConfig.getConfig())); } } return ConfigOrError.fromError( @@ -408,20 +408,14 @@ public String toString() { public static final class PolicySelection { final LoadBalancerProvider provider; - @Deprecated - @Nullable - final Map rawConfig; @Nullable final Object config; - /** Constructs a PolicySelection with selected LB provider, a copy of raw config and the deeply - * parsed LB config. */ + /** Constructs a PolicySelection with selected LB provider and the deeply parsed LB config. */ public PolicySelection( LoadBalancerProvider provider, - @Nullable Map rawConfig, @Nullable Object config) { this.provider = checkNotNull(provider, "provider"); - this.rawConfig = rawConfig; this.config = config; } @@ -444,20 +438,18 @@ public boolean equals(Object o) { } PolicySelection that = (PolicySelection) o; return Objects.equal(provider, that.provider) - && Objects.equal(rawConfig, that.rawConfig) && Objects.equal(config, that.config); } @Override public int hashCode() { - return Objects.hashCode(provider, rawConfig, config); + return Objects.hashCode(provider, config); } @Override public String toString() { return MoreObjects.toStringHelper(this) .add("provider", provider) - .add("rawConfig", rawConfig) .add("config", config) .toString(); } diff --git a/core/src/test/java/io/grpc/internal/AutoConfiguredLoadBalancerFactoryTest.java b/core/src/test/java/io/grpc/internal/AutoConfiguredLoadBalancerFactoryTest.java index 35a5fda02d1..bf7c63818cf 100644 --- a/core/src/test/java/io/grpc/internal/AutoConfiguredLoadBalancerFactoryTest.java +++ b/core/src/test/java/io/grpc/internal/AutoConfiguredLoadBalancerFactoryTest.java @@ -199,7 +199,7 @@ public Subchannel createSubchannel(CreateSubchannelArgs args) { public void handleResolvedAddressGroups_shutsDownOldBalancer() throws Exception { Map serviceConfig = parseConfig("{\"loadBalancingConfig\": [ {\"round_robin\": { } } ] }"); - ConfigOrError lbConfigs = lbf.parseLoadBalancerPolicy(serviceConfig, channelLogger); + ConfigOrError lbConfigs = lbf.parseLoadBalancerPolicy(serviceConfig); final List servers = Collections.singletonList(new EquivalentAddressGroup(new SocketAddress(){})); @@ -243,7 +243,7 @@ public void shutdown() { public void handleResolvedAddressGroups_propagateLbConfigToDelegate() throws Exception { Map rawServiceConfig = parseConfig("{\"loadBalancingConfig\": [ {\"test_lb\": { \"setting1\": \"high\" } } ] }"); - ConfigOrError lbConfigs = lbf.parseLoadBalancerPolicy(rawServiceConfig, channelLogger); + ConfigOrError lbConfigs = lbf.parseLoadBalancerPolicy(rawServiceConfig); assertThat(lbConfigs.getConfig()).isNotNull(); final List servers = @@ -272,7 +272,7 @@ public void handleResolvedAddressGroups_propagateLbConfigToDelegate() throws Exc rawServiceConfig = parseConfig("{\"loadBalancingConfig\": [ {\"test_lb\": { \"setting1\": \"low\" } } ] }"); - lbConfigs = lbf.parseLoadBalancerPolicy(rawServiceConfig, channelLogger); + lbConfigs = lbf.parseLoadBalancerPolicy(rawServiceConfig); handleResult = lb.tryHandleResolvedAddresses( ResolvedAddresses.newBuilder() @@ -297,7 +297,7 @@ public void handleResolvedAddressGroups_propagateLbConfigToDelegate() throws Exc public void handleResolvedAddressGroups_propagateAddrsToDelegate() throws Exception { Map rawServiceConfig = parseConfig("{\"loadBalancingConfig\": [ {\"test_lb\": { \"setting1\": \"high\" } } ] }"); - ConfigOrError lbConfigs = lbf.parseLoadBalancerPolicy(rawServiceConfig, channelLogger); + ConfigOrError lbConfigs = lbf.parseLoadBalancerPolicy(rawServiceConfig); assertThat(lbConfigs.getConfig()).isNotNull(); Helper helper = new TestHelper(); @@ -340,7 +340,7 @@ public void handleResolvedAddressGroups_delegateDoNotAcceptEmptyAddressList_noth Map serviceConfig = parseConfig("{\"loadBalancingConfig\": [ {\"test_lb\": { \"setting1\": \"high\" } } ] }"); - ConfigOrError lbConfig = lbf.parseLoadBalancerPolicy(serviceConfig, helper.getChannelLogger()); + ConfigOrError lbConfig = lbf.parseLoadBalancerPolicy(serviceConfig); Status handleResult = lb.tryHandleResolvedAddresses( ResolvedAddresses.newBuilder() .setAddresses(Collections.emptyList()) @@ -362,7 +362,7 @@ public void handleResolvedAddressGroups_delegateAcceptsEmptyAddressList() Map rawServiceConfig = parseConfig("{\"loadBalancingConfig\": [ {\"test_lb2\": { \"setting1\": \"high\" } } ] }"); ConfigOrError lbConfigs = - lbf.parseLoadBalancerPolicy(rawServiceConfig, helper.getChannelLogger()); + lbf.parseLoadBalancerPolicy(rawServiceConfig); Status handleResult = lb.tryHandleResolvedAddresses( ResolvedAddresses.newBuilder() .setAddresses(Collections.emptyList()) @@ -384,7 +384,7 @@ public void handleResolvedAddressGroups_delegateAcceptsEmptyAddressList() public void handleResolvedAddressGroups_useSelectedLbPolicy() throws Exception { Map rawServiceConfig = parseConfig("{\"loadBalancingConfig\": [{\"round_robin\": {}}]}"); - ConfigOrError lbConfigs = lbf.parseLoadBalancerPolicy(rawServiceConfig, channelLogger); + ConfigOrError lbConfigs = lbf.parseLoadBalancerPolicy(rawServiceConfig); assertThat(lbConfigs.getConfig()).isNotNull(); assertThat(((PolicySelection) lbConfigs.getConfig()).provider.getClass().getName()) .isEqualTo("io.grpc.util.SecretRoundRobinLoadBalancerProvider$Provider"); @@ -471,7 +471,7 @@ public Subchannel createSubchannel(CreateSubchannelArgs args) { nextParsedConfigOrError.set(testLbParsedConfig); Map serviceConfig = parseConfig("{\"loadBalancingConfig\": [ {\"test_lb\": { } } ] }"); - ConfigOrError lbConfigs = lbf.parseLoadBalancerPolicy(serviceConfig, channelLogger); + ConfigOrError lbConfigs = lbf.parseLoadBalancerPolicy(serviceConfig); handleResult = lb.tryHandleResolvedAddresses( ResolvedAddresses.newBuilder() .setAddresses(servers) @@ -494,7 +494,7 @@ public Subchannel createSubchannel(CreateSubchannelArgs args) { testLbParsedConfig = ConfigOrError.fromConfig("bar"); nextParsedConfigOrError.set(testLbParsedConfig); serviceConfig = parseConfig("{\"loadBalancingConfig\": [ {\"test_lb\": { } } ] }"); - lbConfigs = lbf.parseLoadBalancerPolicy(serviceConfig, channelLogger); + lbConfigs = lbf.parseLoadBalancerPolicy(serviceConfig); handleResult = lb.tryHandleResolvedAddresses( ResolvedAddresses.newBuilder() .setAddresses(servers) @@ -512,7 +512,7 @@ public Subchannel createSubchannel(CreateSubchannelArgs args) { public void parseLoadBalancerConfig_failedOnUnknown() throws Exception { Map serviceConfig = parseConfig("{\"loadBalancingConfig\": [ {\"magic_balancer\": {} } ] }"); - ConfigOrError parsed = lbf.parseLoadBalancerPolicy(serviceConfig, channelLogger); + ConfigOrError parsed = lbf.parseLoadBalancerPolicy(serviceConfig); assertThat(parsed.getError()).isNotNull(); assertThat(parsed.getError().getDescription()) .isEqualTo("None of [magic_balancer] specified by Service Config are available."); @@ -522,7 +522,7 @@ public void parseLoadBalancerConfig_failedOnUnknown() throws Exception { public void parseLoadBalancerPolicy_failedOnUnknown() throws Exception { Map serviceConfig = parseConfig("{\"loadBalancingPolicy\": \"magic_balancer\"}"); - ConfigOrError parsed = lbf.parseLoadBalancerPolicy(serviceConfig, channelLogger); + ConfigOrError parsed = lbf.parseLoadBalancerPolicy(serviceConfig); assertThat(parsed.getError()).isNotNull(); assertThat(parsed.getError().getDescription()) .isEqualTo("None of [magic_balancer] specified by Service Config are available."); @@ -535,7 +535,7 @@ public void parseLoadBalancerConfig_multipleValidPolicies() throws Exception { "{\"loadBalancingConfig\": [" + "{\"round_robin\": {}}," + "{\"test_lb\": {} } ] }"); - ConfigOrError parsed = lbf.parseLoadBalancerPolicy(serviceConfig, channelLogger); + ConfigOrError parsed = lbf.parseLoadBalancerPolicy(serviceConfig); assertThat(parsed).isNotNull(); assertThat(parsed.getError()).isNull(); assertThat(parsed.getConfig()).isInstanceOf(PolicySelection.class); @@ -549,7 +549,7 @@ public void parseLoadBalancerConfig_policyShouldBeIgnoredIfConfigExists() throws parseConfig( "{\"loadBalancingConfig\": [{\"round_robin\": {} } ]," + "\"loadBalancingPolicy\": \"pick_first\" }"); - ConfigOrError parsed = lbf.parseLoadBalancerPolicy(serviceConfig, channelLogger); + ConfigOrError parsed = lbf.parseLoadBalancerPolicy(serviceConfig); assertThat(parsed).isNotNull(); assertThat(parsed.getError()).isNull(); assertThat(parsed.getConfig()).isInstanceOf(PolicySelection.class); @@ -564,7 +564,7 @@ public void parseLoadBalancerConfig_policyShouldBeIgnoredEvenIfUnknownPolicyExis parseConfig( "{\"loadBalancingConfig\": [{\"magic_balancer\": {} } ]," + "\"loadBalancingPolicy\": \"round_robin\" }"); - ConfigOrError parsed = lbf.parseLoadBalancerPolicy(serviceConfig, channelLogger); + ConfigOrError parsed = lbf.parseLoadBalancerPolicy(serviceConfig); assertThat(parsed.getError()).isNotNull(); assertThat(parsed.getError().getDescription()) .isEqualTo("None of [magic_balancer] specified by Service Config are available."); @@ -580,7 +580,7 @@ public void parseLoadBalancerConfig_firstInvalidPolicy() throws Exception { "{\"loadBalancingConfig\": [" + "{\"test_lb\": {}}," + "{\"round_robin\": {} } ] }"); - ConfigOrError parsed = lbf.parseLoadBalancerPolicy(serviceConfig, channelLogger); + ConfigOrError parsed = lbf.parseLoadBalancerPolicy(serviceConfig); assertThat(parsed).isNotNull(); assertThat(parsed.getConfig()).isNull(); assertThat(parsed.getError()).isEqualTo(Status.UNKNOWN); @@ -596,7 +596,7 @@ public void parseLoadBalancerConfig_firstValidSecondInvalidPolicy() throws Excep "{\"loadBalancingConfig\": [" + "{\"round_robin\": {}}," + "{\"test_lb\": {} } ] }"); - ConfigOrError parsed = lbf.parseLoadBalancerPolicy(serviceConfig, channelLogger); + ConfigOrError parsed = lbf.parseLoadBalancerPolicy(serviceConfig); assertThat(parsed).isNotNull(); assertThat(parsed.getConfig()).isNotNull(); assertThat(((PolicySelection) parsed.getConfig()).config).isNotNull(); @@ -608,7 +608,7 @@ public void parseLoadBalancerConfig_someProvidesAreNotAvailable() throws Excepti parseConfig("{\"loadBalancingConfig\": [ " + "{\"magic_balancer\": {} }," + "{\"round_robin\": {}} ] }"); - ConfigOrError parsed = lbf.parseLoadBalancerPolicy(serviceConfig, channelLogger); + ConfigOrError parsed = lbf.parseLoadBalancerPolicy(serviceConfig); assertThat(parsed).isNotNull(); assertThat(parsed.getConfig()).isNotNull(); assertThat(((PolicySelection) parsed.getConfig()).config).isNotNull(); @@ -621,7 +621,7 @@ public void parseLoadBalancerConfig_lbConfigPropagated() throws Exception { "{\"loadBalancingConfig\": [" + "{\"grpclb\": {\"childPolicy\": [ {\"pick_first\": {} } ] } }" + "] }"); - ConfigOrError parsed = lbf.parseLoadBalancerPolicy(rawServiceConfig, channelLogger); + ConfigOrError parsed = lbf.parseLoadBalancerPolicy(rawServiceConfig); assertThat(parsed).isNotNull(); assertThat(parsed.getConfig()).isNotNull(); PolicySelection policySelection = (PolicySelection) parsed.getConfig(); diff --git a/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java b/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java index 97208ae8631..65a9ed31e29 100644 --- a/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java +++ b/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java @@ -1163,7 +1163,6 @@ public void nameResolverReturnsEmptySubLists_optionallyAllowed() throws Exceptio rawServiceConfig, new PolicySelection( mockLoadBalancerProvider, - parseConfig(rawLbConfig), parsedLbConfig)); nameResolverFactory.nextConfigOrError.set(ConfigOrError.fromConfig(parsedServiceConfig)); channelBuilder.nameResolverFactory(nameResolverFactory); @@ -2951,7 +2950,7 @@ public void channelTracing_serviceConfigChange() throws Exception { ManagedChannelServiceConfig mcsc1 = createManagedChannelServiceConfig( ImmutableMap.of(), new PolicySelection( - mockLoadBalancerProvider, ImmutableMap.of("foo", "bar"), null)); + mockLoadBalancerProvider, null)); ResolutionResult resolutionResult1 = ResolutionResult.newBuilder() .setAddresses(Collections.singletonList( new EquivalentAddressGroup( @@ -3608,7 +3607,7 @@ public ClientTransportFactory buildClientTransportFactory() { Object fakeLbConfig = new Object(); PolicySelection lbConfigs = new PolicySelection( - mockLoadBalancerProvider, rawServiceConfig, fakeLbConfig); + mockLoadBalancerProvider, fakeLbConfig); mockLoadBalancerProvider.parseLoadBalancingPolicyConfig(rawServiceConfig); ManagedChannelServiceConfig managedChannelServiceConfig = createManagedChannelServiceConfig(rawServiceConfig, lbConfigs); @@ -3766,8 +3765,7 @@ public void nameResolverHelper_emptyConfigSucceeds() { retryEnabled, maxRetryAttemptsLimit, maxHedgedAttemptsLimit, - autoConfiguredLoadBalancerFactory, - mock(ChannelLogger.class)); + autoConfiguredLoadBalancerFactory); ConfigOrError coe = parser.parseServiceConfig(ImmutableMap.of()); @@ -3789,8 +3787,7 @@ public void nameResolverHelper_badConfigFails() { retryEnabled, maxRetryAttemptsLimit, maxHedgedAttemptsLimit, - autoConfiguredLoadBalancerFactory, - mock(ChannelLogger.class)); + autoConfiguredLoadBalancerFactory); ConfigOrError coe = parser.parseServiceConfig(ImmutableMap.of("methodConfig", "bogus")); @@ -3813,8 +3810,7 @@ public void nameResolverHelper_noConfigChosen() { retryEnabled, maxRetryAttemptsLimit, maxHedgedAttemptsLimit, - autoConfiguredLoadBalancerFactory, - mock(ChannelLogger.class)); + autoConfiguredLoadBalancerFactory); ConfigOrError coe = parser.parseServiceConfig(ImmutableMap.of("loadBalancingConfig", ImmutableList.of())); diff --git a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java index 3d1cb31bcc2..8515a83214b 100644 --- a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java @@ -210,8 +210,8 @@ public void onClusterChanged(ClusterUpdate newUpdate) { /* clusterName = */ newUpdate.getClusterName(), /* edsServiceName = */ newUpdate.getEdsServiceName(), /* lrsServerName = */ newUpdate.getLrsServerName(), - new PolicySelection(localityPickingPolicyProvider, null, null /* by EDS policy */), - new PolicySelection(endpointPickingPolicyProvider, null, null)); + new PolicySelection(localityPickingPolicyProvider, null /* by EDS policy */), + new PolicySelection(endpointPickingPolicyProvider, null)); if (isXdsSecurityEnabled()) { updateSslContextProviderSupplier(newUpdate.getUpstreamTlsContext()); } diff --git a/xds/src/main/java/io/grpc/xds/EdsLoadBalancer2.java b/xds/src/main/java/io/grpc/xds/EdsLoadBalancer2.java index 27304346034..a02578530fc 100644 --- a/xds/src/main/java/io/grpc/xds/EdsLoadBalancer2.java +++ b/xds/src/main/java/io/grpc/xds/EdsLoadBalancer2.java @@ -390,7 +390,7 @@ static PriorityLbConfig generatePriorityLbConfig( generateWeightedTargetLbConfig(cluster, edsServiceName, lrsServerName, endpointPickingPolicy, lbRegistry, prioritizedLocalityWeights.get(priority)); PolicySelection childPolicySelection = - new PolicySelection(localityPickingPolicy.getProvider(), null, childConfig); + new PolicySelection(localityPickingPolicy.getProvider(), childConfig); String childName = priorityName(priority); childPolicies.put(childName, childPolicySelection); priorities.add(childName); @@ -413,7 +413,7 @@ static WeightedTargetConfig generateWeightedTargetLbConfig( LrsConfig childConfig = new LrsConfig(cluster, edsServiceName, lrsServerName, locality, endpointPickingPolicy); LoadBalancerProvider childPolicyProvider = lbRegistry.getProvider(LRS_POLICY_NAME); - childPolicy = new PolicySelection(childPolicyProvider, null, childConfig); + childPolicy = new PolicySelection(childPolicyProvider, childConfig); } else { childPolicy = endpointPickingPolicy; } diff --git a/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerProviderTest.java b/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerProviderTest.java index 52404155184..dfef96e5454 100644 --- a/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerProviderTest.java @@ -27,7 +27,6 @@ import io.grpc.internal.ServiceConfigUtil.PolicySelection; import io.grpc.xds.ClusterManagerLoadBalancerProvider.ClusterManagerConfig; import java.io.IOException; -import java.util.Collections; import java.util.Map; import org.junit.Test; import org.junit.runner.RunWith; @@ -132,9 +131,9 @@ public ConfigOrError parseLoadBalancingPolicyConfig( .containsExactly( "child1", new PolicySelection( - lbProviderFoo, Collections.singletonMap("config_name", "config_value"), fooConfig), + lbProviderFoo, fooConfig), "child2", - new PolicySelection(lbProviderBar, Collections.emptyMap(), barConfig)); + new PolicySelection(lbProviderBar, barConfig)); } @Test diff --git a/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerTest.java index 9bd7a8d08fc..9116b6ff1ae 100644 --- a/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerTest.java @@ -259,7 +259,7 @@ private ClusterManagerConfig buildConfig(Map childPolicies) { String childPolicyName = childPolicies.get(name); Object childConfig = lbConfigInventory.get(name); PolicySelection policy = - new PolicySelection(new FakeLoadBalancerProvider(childPolicyName), null, childConfig); + new PolicySelection(new FakeLoadBalancerProvider(childPolicyName), childConfig); childPolicySelections.put(name, policy); } return new ClusterManagerConfig(childPolicySelections); diff --git a/xds/src/test/java/io/grpc/xds/EdsLoadBalancer2Test.java b/xds/src/test/java/io/grpc/xds/EdsLoadBalancer2Test.java index 5b39f255a6f..b964c06157b 100644 --- a/xds/src/test/java/io/grpc/xds/EdsLoadBalancer2Test.java +++ b/xds/src/test/java/io/grpc/xds/EdsLoadBalancer2Test.java @@ -105,9 +105,9 @@ public void uncaughtException(Thread t, Throwable e) { private final FakeClock fakeClock = new FakeClock(); private final LoadBalancerRegistry registry = new LoadBalancerRegistry(); private final PolicySelection roundRobin = - new PolicySelection(new FakeLoadBalancerProvider("round_robin"), null, null); + new PolicySelection(new FakeLoadBalancerProvider("round_robin"), null); private final PolicySelection weightedTarget = - new PolicySelection(new FakeLoadBalancerProvider(WEIGHTED_TARGET_POLICY_NAME), null, null); + new PolicySelection(new FakeLoadBalancerProvider(WEIGHTED_TARGET_POLICY_NAME), null); private final List downstreamBalancers = new ArrayList<>(); private final FakeXdsClient xdsClient = new FakeXdsClient(); private final ObjectPool xdsClientPool = new ObjectPool() { @@ -480,7 +480,7 @@ public void configUpdate_changeEndpointPickingPolicy() { assertThat(leafBalancer.name).isEqualTo("round_robin"); FakeLoadBalancerProvider fakePickFirstProvider = new FakeLoadBalancerProvider("pick_first"); PolicySelection fakePickFirstSelection = - new PolicySelection(fakePickFirstProvider, null, null); + new PolicySelection(fakePickFirstProvider, null); loadBalancer.handleResolvedAddresses(ResolvedAddresses.newBuilder() .setAddresses(Collections.emptyList()) .setAttributes( @@ -633,9 +633,9 @@ private void prepareRealDownstreamLbPolicies(FakeLoadBalancerProvider fakeLeafPo registry.deregister(registry.getProvider(LRS_POLICY_NAME)); registry.register(new LrsLoadBalancerProvider()); PolicySelection weightedTargetSelection = - new PolicySelection(new WeightedTargetLoadBalancerProvider(), null, null); + new PolicySelection(new WeightedTargetLoadBalancerProvider(), null); PolicySelection fakeLeafPolicySelection = - new PolicySelection(fakeLeafPolicyProvider, null, null); + new PolicySelection(fakeLeafPolicyProvider, null); loadBalancer.handleResolvedAddresses( ResolvedAddresses.newBuilder() .setAddresses(Collections.emptyList()) diff --git a/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java index be78db727d6..bc83a873863 100644 --- a/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java @@ -144,9 +144,9 @@ XdsChannel createChannel(List servers) { }; private final PolicySelection fakeLocalityPickingPolicy = - new PolicySelection(mock(LoadBalancerProvider.class), null, null); + new PolicySelection(mock(LoadBalancerProvider.class), null); private final PolicySelection fakeEndpointPickingPolicy = - new PolicySelection(mock(LoadBalancerProvider.class), null, new Object()); + new PolicySelection(mock(LoadBalancerProvider.class), new Object()); @Mock private Helper helper; diff --git a/xds/src/test/java/io/grpc/xds/LrsLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/LrsLoadBalancerTest.java index e53ca3048c7..c1d87d54d01 100644 --- a/xds/src/test/java/io/grpc/xds/LrsLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/LrsLoadBalancerTest.java @@ -149,7 +149,7 @@ public void errorPropagation() { private void deliverResolvedAddresses( List addresses, String childPolicy) { PolicySelection childPolicyConfig = - new PolicySelection(new FakeLoadBalancerProvider(childPolicy), null, null); + new PolicySelection(new FakeLoadBalancerProvider(childPolicy), null); LrsConfig config = new LrsConfig( CLUSTER_NAME, EDS_SERVICE_NAME, LRS_SERVER_NAME, TEST_LOCALITY, childPolicyConfig); diff --git a/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerProviderTest.java b/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerProviderTest.java index a0497252bb8..f0d6958a5b4 100644 --- a/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerProviderTest.java @@ -40,7 +40,7 @@ public class PriorityLoadBalancerProviderTest { @Test public void priorityLbConfig_emptyPriorities() { Map childConfigs = - ImmutableMap.of("p0", new PolicySelection(mock(LoadBalancerProvider.class), null, null)); + ImmutableMap.of("p0", new PolicySelection(mock(LoadBalancerProvider.class), null)); List priorities = ImmutableList.of(); thrown.expect(IllegalArgumentException.class); @@ -51,7 +51,7 @@ public void priorityLbConfig_emptyPriorities() { @Test public void priorityLbConfig_missingChildConfig() { Map childConfigs = - ImmutableMap.of("p1", new PolicySelection(mock(LoadBalancerProvider.class), null, null)); + ImmutableMap.of("p1", new PolicySelection(mock(LoadBalancerProvider.class), null)); List priorities = ImmutableList.of("p0", "p1"); thrown.expect(IllegalArgumentException.class); diff --git a/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerTest.java index 95fd1e95220..390b35a8a21 100644 --- a/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerTest.java @@ -137,11 +137,11 @@ public void handleResolvedAddresses() { Attributes attributes = Attributes.newBuilder().set(Attributes.Key.create("fakeKey"), "fakeValue").build(); Object fooConfig0 = new Object(); - PolicySelection fooPolicy0 = new PolicySelection(fooLbProvider, null, fooConfig0); + PolicySelection fooPolicy0 = new PolicySelection(fooLbProvider, fooConfig0); Object barConfig0 = new Object(); - PolicySelection barPolicy0 = new PolicySelection(barLbProvider, null, barConfig0); + PolicySelection barPolicy0 = new PolicySelection(barLbProvider, barConfig0); Object fooConfig1 = new Object(); - PolicySelection fooPolicy1 = new PolicySelection(fooLbProvider, null, fooConfig1); + PolicySelection fooPolicy1 = new PolicySelection(fooLbProvider, fooConfig1); PriorityLbConfig priorityLbConfig = new PriorityLbConfig( ImmutableMap.of("p0", fooPolicy0, "p1", barPolicy0, "p2", fooPolicy1), @@ -190,7 +190,7 @@ public void handleResolvedAddresses() { newEag = AddressFilter.setPathFilter(newEag, ImmutableList.of("p1")); List newAddresses = ImmutableList.of(newEag); Object newBarConfig = new Object(); - PolicySelection newBarPolicy = new PolicySelection(barLbProvider, null, newBarConfig); + PolicySelection newBarPolicy = new PolicySelection(barLbProvider, newBarConfig); PriorityLbConfig newPriorityLbConfig = new PriorityLbConfig(ImmutableMap.of("p1", newBarPolicy), ImmutableList.of("p1")); priorityLb.handleResolvedAddresses( @@ -217,9 +217,9 @@ public void handleResolvedAddresses() { @Test public void handleNameResolutionError() { Object fooConfig0 = new Object(); - PolicySelection fooPolicy0 = new PolicySelection(fooLbProvider, null, fooConfig0); + PolicySelection fooPolicy0 = new PolicySelection(fooLbProvider, fooConfig0); Object fooConfig1 = new Object(); - PolicySelection fooPolicy1 = new PolicySelection(fooLbProvider, null, fooConfig1); + PolicySelection fooPolicy1 = new PolicySelection(fooLbProvider, fooConfig1); PriorityLbConfig priorityLbConfig = new PriorityLbConfig(ImmutableMap.of("p0", fooPolicy0), ImmutableList.of("p0")); @@ -253,10 +253,10 @@ public void handleNameResolutionError() { @Test public void typicalPriorityFailOverFlow() { - PolicySelection policy0 = new PolicySelection(fooLbProvider, null, new Object()); - PolicySelection policy1 = new PolicySelection(fooLbProvider, null, new Object()); - PolicySelection policy2 = new PolicySelection(fooLbProvider, null, new Object()); - PolicySelection policy3 = new PolicySelection(fooLbProvider, null, new Object()); + PolicySelection policy0 = new PolicySelection(fooLbProvider, new Object()); + PolicySelection policy1 = new PolicySelection(fooLbProvider, new Object()); + PolicySelection policy2 = new PolicySelection(fooLbProvider, new Object()); + PolicySelection policy3 = new PolicySelection(fooLbProvider, new Object()); PriorityLbConfig priorityLbConfig = new PriorityLbConfig( ImmutableMap.of("p0", policy0, "p1", policy1, "p2", policy2, "p3", policy3), diff --git a/xds/src/test/java/io/grpc/xds/WeightedTargetLoadBalancerProviderTest.java b/xds/src/test/java/io/grpc/xds/WeightedTargetLoadBalancerProviderTest.java index a37a06478cb..7a54036b73a 100644 --- a/xds/src/test/java/io/grpc/xds/WeightedTargetLoadBalancerProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/WeightedTargetLoadBalancerProviderTest.java @@ -29,7 +29,6 @@ import io.grpc.internal.ServiceConfigUtil.PolicySelection; import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedPolicySelection; import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedTargetConfig; -import java.util.HashMap; import java.util.Map; import org.junit.Test; import org.junit.runner.RunWith; @@ -129,11 +128,11 @@ public ConfigOrError parseLoadBalancingPolicyConfig(Map rawConfig) { "target_1", new WeightedPolicySelection( 10, - new PolicySelection(lbProviderFoo, new HashMap(), fooConfig)), + new PolicySelection(lbProviderFoo, fooConfig)), "target_2", new WeightedPolicySelection( 20, - new PolicySelection(lbProviderBar, new HashMap(), barConfig))))); + new PolicySelection(lbProviderBar, barConfig))))); assertThat(parsedConfig).isEqualTo(expectedConfig); } } diff --git a/xds/src/test/java/io/grpc/xds/WeightedTargetLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/WeightedTargetLoadBalancerTest.java index b1587bf05eb..55ecfcd41d3 100644 --- a/xds/src/test/java/io/grpc/xds/WeightedTargetLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/WeightedTargetLoadBalancerTest.java @@ -133,13 +133,13 @@ public LoadBalancer newLoadBalancer(Helper helper) { }; private final WeightedPolicySelection weightedLbConfig0 = new WeightedPolicySelection( - weights[0], new PolicySelection(fooLbProvider, null, configs[0])); + weights[0], new PolicySelection(fooLbProvider, configs[0])); private final WeightedPolicySelection weightedLbConfig1 = new WeightedPolicySelection( - weights[1], new PolicySelection(barLbProvider, null, configs[1])); + weights[1], new PolicySelection(barLbProvider, configs[1])); private final WeightedPolicySelection weightedLbConfig2 = new WeightedPolicySelection( - weights[2], new PolicySelection(barLbProvider, null, configs[2])); + weights[2], new PolicySelection(barLbProvider, configs[2])); private final WeightedPolicySelection weightedLbConfig3 = new WeightedPolicySelection( - weights[3], new PolicySelection(fooLbProvider, null, configs[3])); + weights[3], new PolicySelection(fooLbProvider, configs[3])); @Mock private Helper helper; @@ -220,16 +220,16 @@ public void handleResolvedAddresses() { Map newTargets = ImmutableMap.of( "target1", new WeightedPolicySelection( - newWeights[0], new PolicySelection(barLbProvider, null, newConfigs[0])), + newWeights[0], new PolicySelection(barLbProvider, newConfigs[0])), "target2", new WeightedPolicySelection( - newWeights[1], new PolicySelection(barLbProvider, null, newConfigs[1])), + newWeights[1], new PolicySelection(barLbProvider, newConfigs[1])), "target3", new WeightedPolicySelection( - newWeights[2], new PolicySelection(fooLbProvider, null, newConfigs[2])), + newWeights[2], new PolicySelection(fooLbProvider, newConfigs[2])), "target4", new WeightedPolicySelection( - newWeights[3], new PolicySelection(fooLbProvider, null, newConfigs[3]))); + newWeights[3], new PolicySelection(fooLbProvider, newConfigs[3]))); weightedTargetLb.handleResolvedAddresses( ResolvedAddresses.newBuilder() .setAddresses(ImmutableList.of()) diff --git a/xds/src/test/java/io/grpc/xds/XdsRoutingLoadBalancerProviderTest.java b/xds/src/test/java/io/grpc/xds/XdsRoutingLoadBalancerProviderTest.java index d600bafd69e..ec8c73fc722 100644 --- a/xds/src/test/java/io/grpc/xds/XdsRoutingLoadBalancerProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsRoutingLoadBalancerProviderTest.java @@ -34,7 +34,6 @@ import io.grpc.xds.XdsRoutingLoadBalancerProvider.XdsRoutingConfig; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import org.junit.Test; @@ -224,9 +223,9 @@ public ConfigOrError parseLoadBalancingPolicyConfig(Map rawConfig) { assertThat(configActions).hasSize(2); assertThat(configActions).containsExactly( "action_foo", - new PolicySelection(lbProviderFoo, new HashMap(), fooConfig), + new PolicySelection(lbProviderFoo, fooConfig), "action_bar", new PolicySelection( - lbProviderBar, new HashMap(), barConfig)); + lbProviderBar, barConfig)); } } diff --git a/xds/src/test/java/io/grpc/xds/XdsRoutingLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/XdsRoutingLoadBalancerTest.java index f9bc47fc55e..10b11398bc8 100644 --- a/xds/src/test/java/io/grpc/xds/XdsRoutingLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsRoutingLoadBalancerTest.java @@ -334,7 +334,7 @@ private XdsRoutingConfig buildConfig(Map childPolicies) { String childPolicyName = childPolicies.get(route); Object childConfig = lbConfigInventory.get(childActionName); PolicySelection policy = - new PolicySelection(new FakeLoadBalancerProvider(childPolicyName), null, childConfig); + new PolicySelection(new FakeLoadBalancerProvider(childPolicyName), childConfig); childPolicySelections.put(childActionName, policy); routeList.add(route); } From 147bee81425f1b28940a31c0dc6a5053935252fa Mon Sep 17 00:00:00 2001 From: sanjaypujare Date: Wed, 23 Sep 2020 10:35:02 -0700 Subject: [PATCH 62/86] xds: implement fallbackProtocolNegotiator for XdsChannelBuilder (#7438) --- .../internal/sds/SdsProtocolNegotiators.java | 30 +++++++++++++++---- .../xds/internal/sds/XdsChannelBuilder.java | 12 +++++++- .../io/grpc/xds/XdsSdsClientServerTest.java | 3 +- .../sds/SdsProtocolNegotiatorsTest.java | 26 ++++++++++++---- 4 files changed, 59 insertions(+), 12 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/internal/sds/SdsProtocolNegotiators.java b/xds/src/main/java/io/grpc/xds/internal/sds/SdsProtocolNegotiators.java index d28b5c82f60..ea5a1435d86 100644 --- a/xds/src/main/java/io/grpc/xds/internal/sds/SdsProtocolNegotiators.java +++ b/xds/src/main/java/io/grpc/xds/internal/sds/SdsProtocolNegotiators.java @@ -59,9 +59,14 @@ private SdsProtocolNegotiators() { private static final AsciiString SCHEME = AsciiString.of("http"); - /** Returns a {@link ProtocolNegotiatorFactory} to be used on {@link NettyChannelBuilder}. */ - public static ProtocolNegotiatorFactory clientProtocolNegotiatorFactory() { - return new ClientSdsProtocolNegotiatorFactory(); + /** + * Returns a {@link ProtocolNegotiatorFactory} to be used on {@link NettyChannelBuilder}. + * + * @param fallbackNegotiator protocol negotiator to use as fallback. + */ + public static ProtocolNegotiatorFactory clientProtocolNegotiatorFactory( + @Nullable ProtocolNegotiator fallbackNegotiator) { + return new ClientSdsProtocolNegotiatorFactory(fallbackNegotiator); } /** @@ -80,9 +85,16 @@ public static ServerSdsProtocolNegotiator serverProtocolNegotiator(int port, private static final class ClientSdsProtocolNegotiatorFactory implements InternalNettyChannelBuilder.ProtocolNegotiatorFactory { + private final ProtocolNegotiator fallbackProtocolNegotiator; + + private ClientSdsProtocolNegotiatorFactory(ProtocolNegotiator fallbackNegotiator) { + this.fallbackProtocolNegotiator = fallbackNegotiator; + } + @Override public InternalProtocolNegotiator.ProtocolNegotiator buildProtocolNegotiator() { - final ClientSdsProtocolNegotiator negotiator = new ClientSdsProtocolNegotiator(); + final ClientSdsProtocolNegotiator negotiator = + new ClientSdsProtocolNegotiator(fallbackProtocolNegotiator); final class LocalSdsNegotiator implements InternalProtocolNegotiator.ProtocolNegotiator { @Override @@ -108,6 +120,12 @@ public void close() { @VisibleForTesting static final class ClientSdsProtocolNegotiator implements ProtocolNegotiator { + @Nullable private final ProtocolNegotiator fallbackProtocolNegotiator; + + ClientSdsProtocolNegotiator(@Nullable ProtocolNegotiator fallbackProtocolNegotiator) { + this.fallbackProtocolNegotiator = fallbackProtocolNegotiator; + } + @Override public AsciiString scheme() { return SCHEME; @@ -119,7 +137,9 @@ public ChannelHandler newHandler(GrpcHttp2ConnectionHandler grpcHandler) { SslContextProviderSupplier localSslContextProviderSupplier = grpcHandler.getEagAttributes().get(XdsAttributes.ATTR_SSL_CONTEXT_PROVIDER_SUPPLIER); if (localSslContextProviderSupplier == null) { - return InternalProtocolNegotiators.plaintext().newHandler(grpcHandler); + checkNotNull( + fallbackProtocolNegotiator, "No TLS config and no fallbackProtocolNegotiator!"); + return fallbackProtocolNegotiator.newHandler(grpcHandler); } return new ClientSdsHandler(grpcHandler, localSslContextProviderSupplier); } diff --git a/xds/src/main/java/io/grpc/xds/internal/sds/XdsChannelBuilder.java b/xds/src/main/java/io/grpc/xds/internal/sds/XdsChannelBuilder.java index 2c80fc9da89..ab0384c42e9 100644 --- a/xds/src/main/java/io/grpc/xds/internal/sds/XdsChannelBuilder.java +++ b/xds/src/main/java/io/grpc/xds/internal/sds/XdsChannelBuilder.java @@ -20,6 +20,7 @@ import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.grpc.netty.InternalNettyChannelBuilder; +import io.grpc.netty.InternalProtocolNegotiator; import io.grpc.netty.NettyChannelBuilder; import java.net.SocketAddress; import javax.annotation.CheckReturnValue; @@ -31,6 +32,7 @@ public final class XdsChannelBuilder extends ForwardingChannelBuilder { private final NettyChannelBuilder delegate; + private InternalProtocolNegotiator.ProtocolNegotiator fallbackProtocolNegotiator; private XdsChannelBuilder(NettyChannelBuilder delegate) { this.delegate = delegate; @@ -63,6 +65,13 @@ public static XdsChannelBuilder forTarget(String target) { return new XdsChannelBuilder(NettyChannelBuilder.forTarget(target)); } + /** Set the fallback protocolNegotiator. Pass null to unset a previously set value. */ + public XdsChannelBuilder fallbackProtocolNegotiator( + InternalProtocolNegotiator.ProtocolNegotiator fallbackProtocolNegotiator) { + this.fallbackProtocolNegotiator = fallbackProtocolNegotiator; + return this; + } + @Override protected ManagedChannelBuilder delegate() { return delegate; @@ -71,7 +80,8 @@ protected ManagedChannelBuilder delegate() { @Override public ManagedChannel build() { InternalNettyChannelBuilder.setProtocolNegotiatorFactory( - delegate, SdsProtocolNegotiators.clientProtocolNegotiatorFactory()); + delegate, + SdsProtocolNegotiators.clientProtocolNegotiatorFactory(fallbackProtocolNegotiator)); return delegate.build(); } } diff --git a/xds/src/test/java/io/grpc/xds/XdsSdsClientServerTest.java b/xds/src/test/java/io/grpc/xds/XdsSdsClientServerTest.java index 0a16e3f60e5..98932b97822 100644 --- a/xds/src/test/java/io/grpc/xds/XdsSdsClientServerTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsSdsClientServerTest.java @@ -358,7 +358,8 @@ private SimpleServiceGrpc.SimpleServiceBlockingStub getBlockingStub( fakeNameResolverFactory = new FakeNameResolverFactory.Builder(expectedUri).build(); NameResolverRegistry.getDefaultRegistry().register(fakeNameResolverFactory); XdsChannelBuilder channelBuilder = - XdsChannelBuilder.forTarget("sdstest://localhost:" + port); + XdsChannelBuilder.forTarget("sdstest://localhost:" + port) + .fallbackProtocolNegotiator(InternalProtocolNegotiators.plaintext()); if (overrideAuthority != null) { channelBuilder = channelBuilder.overrideAuthority(overrideAuthority); } diff --git a/xds/src/test/java/io/grpc/xds/internal/sds/SdsProtocolNegotiatorsTest.java b/xds/src/test/java/io/grpc/xds/internal/sds/SdsProtocolNegotiatorsTest.java index db82bcdb6b9..b69109f2e70 100644 --- a/xds/src/test/java/io/grpc/xds/internal/sds/SdsProtocolNegotiatorsTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/sds/SdsProtocolNegotiatorsTest.java @@ -148,12 +148,27 @@ private static CommonTlsContext getCommonTlsContext( @Test public void clientSdsProtocolNegotiatorNewHandler_noTlsContextAttribute() { - ClientSdsProtocolNegotiator pn = new ClientSdsProtocolNegotiator(); + ChannelHandler mockChannelHandler = mock(ChannelHandler.class); + ProtocolNegotiator mockProtocolNegotiator = mock(ProtocolNegotiator.class); + when(mockProtocolNegotiator.newHandler(grpcHandler)).thenReturn(mockChannelHandler); + ClientSdsProtocolNegotiator pn = new ClientSdsProtocolNegotiator(mockProtocolNegotiator); ChannelHandler newHandler = pn.newHandler(grpcHandler); assertThat(newHandler).isNotNull(); - // ProtocolNegotiators.WaitUntilActiveHandler not accessible, get canonical name - assertThat(newHandler.getClass().getCanonicalName()) - .contains("io.grpc.netty.ProtocolNegotiators.WaitUntilActiveHandler"); + assertThat(newHandler).isSameInstanceAs(mockChannelHandler); + } + + @Test + public void clientSdsProtocolNegotiatorNewHandler_noFallback_expectException() { + ClientSdsProtocolNegotiator pn = + new ClientSdsProtocolNegotiator(/* fallbackProtocolNegotiator= */ null); + try { + pn.newHandler(grpcHandler); + fail("exception expected!"); + } catch (NullPointerException expected) { + assertThat(expected) + .hasMessageThat() + .contains("No TLS config and no fallbackProtocolNegotiator!"); + } } @Test @@ -161,7 +176,8 @@ public void clientSdsProtocolNegotiatorNewHandler_withTlsContextAttribute() { UpstreamTlsContext upstreamTlsContext = CommonTlsContextTestsUtil.buildUpstreamTlsContext( getCommonTlsContext(/* tlsCertificate= */ null, /* certContext= */ null)); - ClientSdsProtocolNegotiator pn = new ClientSdsProtocolNegotiator(); + ClientSdsProtocolNegotiator pn = + new ClientSdsProtocolNegotiator(InternalProtocolNegotiators.plaintext()); GrpcHttp2ConnectionHandler mockHandler = mock(GrpcHttp2ConnectionHandler.class); TlsContextManager mockTlsContextManager = mock(TlsContextManager.class); when(mockHandler.getEagAttributes()) From f0552005661d000dfadb8db70b0cd5106167758b Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Wed, 23 Sep 2020 17:00:41 -0700 Subject: [PATCH 63/86] xds: define individual LDS/RDS data watch interface (#7453) --- .../main/java/io/grpc/xds/EnvoyProtoData.java | 47 ++++++ xds/src/main/java/io/grpc/xds/XdsClient.java | 139 ++++++++++++++++++ 2 files changed, 186 insertions(+) diff --git a/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java b/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java index 2516ab17ca2..2917ea009fa 100644 --- a/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java +++ b/xds/src/main/java/io/grpc/xds/EnvoyProtoData.java @@ -59,6 +59,7 @@ * gRPC. If the protobuf message contains invalid data, the conversion should fail and no object * should be instantiated. */ +// TODO(chengyuanzhang): put data types into smaller categories. final class EnvoyProtoData { // Prevent instantiation. @@ -839,6 +840,52 @@ public String toString() { } } + /** See corresponding Envoy proto message {@link + * io.envoyproxy.envoy.config.route.v3.VirtualHost}. */ + static final class VirtualHost { + // Canonical name of this virtual host. + private final String name; + // A list of domains (host/authority header) that will be matched to this virtual host. + private final List domains; + // The list of routes that will be matched, in order, for incoming requests. + private final List routes; + + private VirtualHost(String name, List domains, List routes) { + this.name = name; + this.domains = domains; + this.routes = routes; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("name", name) + .add("domains", domains) + .add("routes", routes) + .toString(); + } + + static StructOrError fromEnvoyProtoVirtualHost( + io.envoyproxy.envoy.config.route.v3.VirtualHost proto) { + String name = proto.getName(); + List routes = new ArrayList<>(proto.getRoutesCount()); + for (io.envoyproxy.envoy.config.route.v3.Route routeProto : proto.getRoutesList()) { + StructOrError route = Route.fromEnvoyProtoRoute(routeProto); + if (route == null) { + continue; + } + if (route.getErrorDetail() != null) { + return StructOrError.fromError( + "Virtual host [" + name + "] contains invalid route : " + route.getErrorDetail()); + } + routes.add(route.getStruct()); + } + return StructOrError.fromStruct( + new VirtualHost(name, Collections.unmodifiableList(proto.getDomainsList()), + Collections.unmodifiableList(routes))); + } + } + /** See corresponding Envoy proto message {@link io.envoyproxy.envoy.config.route.v3.Route}. */ static final class Route { private final RouteMatch routeMatch; diff --git a/xds/src/main/java/io/grpc/xds/XdsClient.java b/xds/src/main/java/io/grpc/xds/XdsClient.java index 3e639c7fa3d..606a47f9068 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClient.java +++ b/xds/src/main/java/io/grpc/xds/XdsClient.java @@ -20,6 +20,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; +import com.google.common.base.MoreObjects.ToStringHelper; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -30,6 +31,7 @@ import io.grpc.xds.EnvoyProtoData.Locality; import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints; import io.grpc.xds.EnvoyProtoData.Route; +import io.grpc.xds.EnvoyProtoData.VirtualHost; import io.grpc.xds.EnvoyServerProtoData.Listener; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import io.grpc.xds.LoadStatsManager.LoadStatsStore; @@ -56,6 +58,7 @@ abstract class XdsClient { * traffic mirroring, retry or hedging, default timeouts and load balancing policy that will * be used to generate a service config. */ + // TODO(chengyuanzhang): delete me. static final class ConfigUpdate { private final List routes; @@ -100,6 +103,109 @@ ConfigUpdate build() { } } + static final class LdsUpdate { + // Total number of nanoseconds to keep alive an HTTP request/response stream. + private final long httpMaxStreamDurationNano; + // The name of the route configuration to be used for RDS resource discovery. + @Nullable + private final String rdsName; + // The list virtual hosts that make up the route table. + @Nullable + private final List virtualHosts; + + private LdsUpdate(long httpMaxStreamDurationNano, @Nullable String rdsName, + @Nullable List virtualHosts) { + this.httpMaxStreamDurationNano = httpMaxStreamDurationNano; + this.rdsName = rdsName; + this.virtualHosts = virtualHosts; + } + + long getHttpMaxStreamDurationNano() { + return httpMaxStreamDurationNano; + } + + @Nullable + String getRdsName() { + return rdsName; + } + + @Nullable + List getVirtualHosts() { + return virtualHosts; + } + + @Override + public String toString() { + ToStringHelper toStringHelper = MoreObjects.toStringHelper(this); + toStringHelper.add("httpMaxStreamDurationNano", httpMaxStreamDurationNano); + if (rdsName != null) { + toStringHelper.add("rdsName", rdsName); + } else { + toStringHelper.add("virtualHosts", virtualHosts); + } + return toStringHelper.toString(); + } + + static Builder newBuilder() { + return new Builder(); + } + + private static class Builder { + private long httpMaxStreamDurationNano; + @Nullable + private String rdsName; + @Nullable + private List virtualHosts; + + private Builder() { + } + + Builder setHttpMaxStreamDurationNano(long httpMaxStreamDurationNano) { + this.httpMaxStreamDurationNano = httpMaxStreamDurationNano; + return this; + } + + Builder setRdsName(String rdsName) { + this.rdsName = rdsName; + return this; + } + + Builder setVirtualHosts(List virtualHosts) { + this.virtualHosts = virtualHosts; + return this; + } + + LdsUpdate build() { + checkState((rdsName == null) != (virtualHosts == null), "one of rdsName and virtualHosts"); + return new LdsUpdate(httpMaxStreamDurationNano, rdsName, virtualHosts); + } + } + } + + static final class RdsUpdate { + // The list virtual hosts that make up the route table. + private final List virtualHosts; + + private RdsUpdate(List virtualHosts) { + this.virtualHosts = virtualHosts; + } + + static RdsUpdate fromVirtualHosts(List virtualHosts) { + return new RdsUpdate(virtualHosts); + } + + List getVirtualHosts() { + return virtualHosts; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("virtualHosts", virtualHosts) + .toString(); + } + } + /** * Data class containing the results of performing a resource discovery RPC via CDS protocol. * The results include configurations for a single upstream cluster, such as endpoint discovery @@ -422,6 +528,14 @@ private interface ResourceWatcher { void onResourceDoesNotExist(String resourceName); } + interface LdsResourceWatcher extends ResourceWatcher { + void onChanged(LdsUpdate update); + } + + interface RdsResourceWatcher extends ResourceWatcher { + void onChanged(RdsUpdate update); + } + /** * Config watcher interface. To be implemented by the xDS resolver. */ @@ -476,9 +590,34 @@ interface ListenerWatcher extends ResourceWatcher { * targets for. * @param watcher the {@link ConfigWatcher} to receive {@link ConfigUpdate}. */ + // TODO(chengyuanzhang): delete me. void watchConfigData(String targetAuthority, ConfigWatcher watcher) { } + /** + * Registers a data watcher for the given LDS resource. + */ + void watchLdsResource(String resourceName, LdsResourceWatcher watcher) { + } + + /** + * Unregisters the given LDS resource watcher. + */ + void cancelLdsResourceWatch(String resourceName, LdsResourceWatcher watcher) { + } + + /** + * Registers a data watcher for the given RDS resource. + */ + void watchRdsResource(String resourceName, RdsResourceWatcher watcher) { + } + + /** + * Unregisters the given RDS resource watcher. + */ + void cancelRdsResourceWatch(String resourceName, RdsResourceWatcher watcher) { + } + /** * Registers a data watcher for the given cluster. */ From f62742561de6e1d503e2c59c65832ecf6e995611 Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Wed, 23 Sep 2020 17:06:55 -0700 Subject: [PATCH 64/86] xds: add toString() and delete unnecessary equals()/hashCode() for LB configs (#7451) --- .../main/java/io/grpc/xds/AddressFilter.java | 5 ++++ .../io/grpc/xds/CdsLoadBalancerProvider.java | 18 +++---------- .../io/grpc/xds/EdsLoadBalancerProvider.java | 25 ------------------- .../io/grpc/xds/LrsLoadBalancerProvider.java | 12 +++++++++ 4 files changed, 20 insertions(+), 40 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/AddressFilter.java b/xds/src/main/java/io/grpc/xds/AddressFilter.java index 62da7ada186..841e96d06bb 100644 --- a/xds/src/main/java/io/grpc/xds/AddressFilter.java +++ b/xds/src/main/java/io/grpc/xds/AddressFilter.java @@ -82,5 +82,10 @@ private static final class PathChain { PathChain(String name) { this.name = checkNotNull(name, "name"); } + + @Override + public String toString() { + return name + (next == null ? "" : ", " + next); + } } } diff --git a/xds/src/main/java/io/grpc/xds/CdsLoadBalancerProvider.java b/xds/src/main/java/io/grpc/xds/CdsLoadBalancerProvider.java index bf1dbb5bc46..7e289270681 100644 --- a/xds/src/main/java/io/grpc/xds/CdsLoadBalancerProvider.java +++ b/xds/src/main/java/io/grpc/xds/CdsLoadBalancerProvider.java @@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument; +import com.google.common.base.MoreObjects; import io.grpc.Internal; import io.grpc.LoadBalancer; import io.grpc.LoadBalancer.Helper; @@ -26,7 +27,6 @@ import io.grpc.Status; import io.grpc.internal.JsonUtil; import java.util.Map; -import java.util.Objects; /** * The provider for the "cds" balancing policy. This class should not be directly referenced in @@ -96,20 +96,8 @@ static final class CdsConfig { } @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - CdsConfig cdsConfig = (CdsConfig) o; - return Objects.equals(name, cdsConfig.name); - } - - @Override - public int hashCode() { - return Objects.hash(name); + public String toString() { + return MoreObjects.toStringHelper(this).add("name", name).toString(); } } } diff --git a/xds/src/main/java/io/grpc/xds/EdsLoadBalancerProvider.java b/xds/src/main/java/io/grpc/xds/EdsLoadBalancerProvider.java index 57830954036..3130c42b708 100644 --- a/xds/src/main/java/io/grpc/xds/EdsLoadBalancerProvider.java +++ b/xds/src/main/java/io/grpc/xds/EdsLoadBalancerProvider.java @@ -19,7 +19,6 @@ import static com.google.common.base.Preconditions.checkNotNull; import com.google.common.base.MoreObjects; -import com.google.common.base.Objects; import io.grpc.Internal; import io.grpc.LoadBalancer; import io.grpc.LoadBalancer.Helper; @@ -95,29 +94,5 @@ public String toString() { .add("endpointPickingPolicy", endpointPickingPolicy) .toString(); } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof EdsConfig)) { - return false; - } - EdsConfig that = (EdsConfig) obj; - return Objects.equal(this.clusterName, that.clusterName) - && Objects.equal(this.edsServiceName, that.edsServiceName) - && Objects.equal(this.lrsServerName, that.lrsServerName) - && Objects.equal(this.localityPickingPolicy, that.localityPickingPolicy) - && Objects.equal(this.endpointPickingPolicy, that.endpointPickingPolicy); - } - - @Override - public int hashCode() { - return - Objects.hashCode( - clusterName, - edsServiceName, - lrsServerName, - localityPickingPolicy, - endpointPickingPolicy); - } } } diff --git a/xds/src/main/java/io/grpc/xds/LrsLoadBalancerProvider.java b/xds/src/main/java/io/grpc/xds/LrsLoadBalancerProvider.java index d4a663b261e..a7e6e8a9df4 100644 --- a/xds/src/main/java/io/grpc/xds/LrsLoadBalancerProvider.java +++ b/xds/src/main/java/io/grpc/xds/LrsLoadBalancerProvider.java @@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkNotNull; +import com.google.common.base.MoreObjects; import io.grpc.Internal; import io.grpc.LoadBalancer; import io.grpc.LoadBalancerProvider; @@ -80,5 +81,16 @@ static final class LrsConfig { this.locality = checkNotNull(locality, "locality"); this.childPolicy = checkNotNull(childPolicy, "childPolicy"); } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("clusterName", clusterName) + .add("edsServiceName", edsServiceName) + .add("lrsServerName", lrsServerName) + .add("locality", locality) + .add("childPolicy", childPolicy) + .toString(); + } } } From 41ba2427823db6cb180ef3d4a175931b418b56cb Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Wed, 23 Sep 2020 17:21:18 -0700 Subject: [PATCH 65/86] xds: delete old EdsLoadBalancer codepath (#7448) --- .../java/io/grpc/xds/EdsLoadBalancer.java | 371 ----- .../main/java/io/grpc/xds/LocalityStore.java | 578 ------- .../main/java/io/grpc/xds/XdsLbPolicies.java | 2 - .../io/grpc/xds/XdsRoutingLoadBalancer.java | 285 ---- .../xds/XdsRoutingLoadBalancerProvider.java | 348 ----- .../services/io.grpc.LoadBalancerProvider | 1 - .../java/io/grpc/xds/EdsLoadBalancerTest.java | 866 ----------- .../java/io/grpc/xds/LocalityStoreTest.java | 1378 ----------------- .../XdsRoutingLoadBalancerProviderTest.java | 231 --- .../grpc/xds/XdsRoutingLoadBalancerTest.java | 478 ------ 10 files changed, 4538 deletions(-) delete mode 100644 xds/src/main/java/io/grpc/xds/EdsLoadBalancer.java delete mode 100644 xds/src/main/java/io/grpc/xds/LocalityStore.java delete mode 100644 xds/src/main/java/io/grpc/xds/XdsRoutingLoadBalancer.java delete mode 100644 xds/src/main/java/io/grpc/xds/XdsRoutingLoadBalancerProvider.java delete mode 100644 xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java delete mode 100644 xds/src/test/java/io/grpc/xds/LocalityStoreTest.java delete mode 100644 xds/src/test/java/io/grpc/xds/XdsRoutingLoadBalancerProviderTest.java delete mode 100644 xds/src/test/java/io/grpc/xds/XdsRoutingLoadBalancerTest.java diff --git a/xds/src/main/java/io/grpc/xds/EdsLoadBalancer.java b/xds/src/main/java/io/grpc/xds/EdsLoadBalancer.java deleted file mode 100644 index 60652af299f..00000000000 --- a/xds/src/main/java/io/grpc/xds/EdsLoadBalancer.java +++ /dev/null @@ -1,371 +0,0 @@ -/* - * Copyright 2019 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.xds; - -import static com.google.common.base.Preconditions.checkNotNull; -import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import io.grpc.Attributes; -import io.grpc.InternalLogId; -import io.grpc.LoadBalancer; -import io.grpc.LoadBalancerRegistry; -import io.grpc.Status; -import io.grpc.internal.ExponentialBackoffPolicy; -import io.grpc.internal.GrpcUtil; -import io.grpc.internal.ObjectPool; -import io.grpc.util.GracefulSwitchLoadBalancer; -import io.grpc.xds.Bootstrapper.BootstrapInfo; -import io.grpc.xds.EdsLoadBalancerProvider.EdsConfig; -import io.grpc.xds.EnvoyProtoData.DropOverload; -import io.grpc.xds.EnvoyProtoData.Locality; -import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints; -import io.grpc.xds.EnvoyProtoData.Node; -import io.grpc.xds.LoadStatsManager.LoadStatsStore; -import io.grpc.xds.LocalityStore.LocalityStoreFactory; -import io.grpc.xds.XdsClient.EndpointUpdate; -import io.grpc.xds.XdsClient.EndpointWatcher; -import io.grpc.xds.XdsClient.RefCountedXdsClientObjectPool; -import io.grpc.xds.XdsClient.XdsChannel; -import io.grpc.xds.XdsClient.XdsClientFactory; -import io.grpc.xds.XdsLogger.XdsLogLevel; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import javax.annotation.Nullable; - -/** Load balancer for the EDS LB policy. */ -final class EdsLoadBalancer extends LoadBalancer { - - private final InternalLogId logId; - private final XdsLogger logger; - private final GracefulSwitchLoadBalancer switchingLoadBalancer; - private final LoadBalancerRegistry lbRegistry; - private final LocalityStoreFactory localityStoreFactory; - private final Bootstrapper bootstrapper; - private final XdsChannelFactory channelFactory; - private final Helper helper; - - @Nullable - private ObjectPool xdsClientPool; - @Nullable - private XdsClient xdsClient; - @Nullable - private String clusterName; - @Nullable - private String edsServiceName; - - EdsLoadBalancer(Helper helper) { - this( - helper, - LoadBalancerRegistry.getDefaultRegistry(), - LocalityStoreFactory.getInstance(), - Bootstrapper.getInstance(), - XdsChannelFactory.getInstance()); - } - - @VisibleForTesting - EdsLoadBalancer( - Helper helper, - LoadBalancerRegistry lbRegistry, - LocalityStoreFactory localityStoreFactory, - Bootstrapper bootstrapper, - XdsChannelFactory channelFactory) { - this.helper = checkNotNull(helper, "helper"); - this.lbRegistry = checkNotNull(lbRegistry, "lbRegistry"); - this.localityStoreFactory = checkNotNull(localityStoreFactory, "localityStoreFactory"); - this.bootstrapper = checkNotNull(bootstrapper, "bootstrapper"); - this.channelFactory = checkNotNull(channelFactory, "channelFactory"); - this.switchingLoadBalancer = new GracefulSwitchLoadBalancer(helper); - logId = InternalLogId.allocate("eds-lb", helper.getAuthority()); - logger = XdsLogger.withLogId(logId); - logger.log(XdsLogLevel.INFO, "Created"); - } - - @Override - public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { - logger.log(XdsLogLevel.DEBUG, "Received resolution result: {0}", resolvedAddresses); - Object lbConfig = resolvedAddresses.getLoadBalancingPolicyConfig(); - if (lbConfig == null) { - handleNameResolutionError(Status.UNAVAILABLE.withDescription("Missing EDS lb config")); - return; - } - EdsConfig newEdsConfig = (EdsConfig) lbConfig; - if (logger.isLoggable(XdsLogLevel.INFO)) { - logger.log( - XdsLogLevel.INFO, - "Received EDS lb config: cluster={0}, child_policy={1}, " - + "eds_service_name={2}, report_load={3}", - newEdsConfig.clusterName, - newEdsConfig.endpointPickingPolicy.getProvider().getPolicyName(), - newEdsConfig.edsServiceName, - newEdsConfig.lrsServerName != null); - } - boolean firstUpdate = false; - if (clusterName == null) { - firstUpdate = true; - } - clusterName = newEdsConfig.clusterName; - if (xdsClientPool == null) { - Attributes attributes = resolvedAddresses.getAttributes(); - xdsClientPool = attributes.get(XdsAttributes.XDS_CLIENT_POOL); - if (xdsClientPool == null) { - final BootstrapInfo bootstrapInfo; - final XdsChannel channel; - try { - bootstrapInfo = bootstrapper.readBootstrap(); - channel = channelFactory.createChannel(bootstrapInfo.getServers()); - } catch (Exception e) { - helper.updateBalancingState( - TRANSIENT_FAILURE, - new ErrorPicker( - Status.UNAVAILABLE.withDescription("Failed to bootstrap").withCause(e))); - return; - } - - final Node node = bootstrapInfo.getNode(); - XdsClientFactory xdsClientFactory = new XdsClientFactory() { - @Override - XdsClient createXdsClient() { - return - new XdsClientImpl( - helper.getAuthority(), - channel, - node, - helper.getSynchronizationContext(), - helper.getScheduledExecutorService(), - new ExponentialBackoffPolicy.Provider(), - GrpcUtil.STOPWATCH_SUPPLIER); - } - }; - xdsClientPool = new RefCountedXdsClientObjectPool(xdsClientFactory); - } else { - logger.log(XdsLogLevel.INFO, "Use xDS client from channel"); - } - xdsClient = xdsClientPool.getObject(); - } - - // Note: childPolicy change will be handled in LocalityStore, to be implemented. - // If edsServiceName in XdsConfig is changed, do a graceful switch. - if (firstUpdate || !Objects.equals(newEdsConfig.edsServiceName, edsServiceName)) { - LoadBalancer.Factory clusterEndpointsLoadBalancerFactory = - new ClusterEndpointsBalancerFactory(newEdsConfig.edsServiceName); - switchingLoadBalancer.switchTo(clusterEndpointsLoadBalancerFactory); - } - switchingLoadBalancer.handleResolvedAddresses(resolvedAddresses); - this.edsServiceName = newEdsConfig.edsServiceName; - } - - @Override - public void handleNameResolutionError(Status error) { - logger.log(XdsLogLevel.WARNING, "Received name resolution error: {0}", error); - // This will go into TRANSIENT_FAILURE if we have not yet received any endpoint update and - // otherwise keep running with the data we had previously. - switchingLoadBalancer.handleNameResolutionError(error); - } - - @Override - public boolean canHandleEmptyAddressListFromNameResolution() { - return true; - } - - @Override - public void shutdown() { - logger.log(XdsLogLevel.INFO, "Shutdown"); - switchingLoadBalancer.shutdown(); - if (xdsClient != null) { - xdsClient = xdsClientPool.returnObject(xdsClient); - } - } - - /** - * A load balancer factory that provides a load balancer for a given cluster service. - */ - private final class ClusterEndpointsBalancerFactory extends LoadBalancer.Factory { - @Nullable final String clusterServiceName; - - ClusterEndpointsBalancerFactory(@Nullable String clusterServiceName) { - this.clusterServiceName = clusterServiceName; - } - - @Override - public LoadBalancer newLoadBalancer(Helper helper) { - return new ClusterEndpointsBalancer(helper); - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof ClusterEndpointsBalancerFactory)) { - return false; - } - ClusterEndpointsBalancerFactory that = (ClusterEndpointsBalancerFactory) o; - return Objects.equals(clusterServiceName, that.clusterServiceName); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), clusterServiceName); - } - - /** - * Load-balances endpoints for a given cluster. - */ - final class ClusterEndpointsBalancer extends LoadBalancer { - // Name of the resource to be used for querying endpoint information. - final String resourceName; - final Helper helper; - final EndpointWatcherImpl endpointWatcher; - final LocalityStore localityStore; - boolean isReportingLoad; - - ClusterEndpointsBalancer(Helper helper) { - this.helper = helper; - resourceName = clusterServiceName != null ? clusterServiceName : clusterName; - LoadStatsStore loadStatsStore = xdsClient.addClientStats(clusterName, clusterServiceName); - localityStore = - localityStoreFactory.newLocalityStore(logId, helper, lbRegistry, loadStatsStore); - endpointWatcher = new EndpointWatcherImpl(); - logger.log( - XdsLogLevel.INFO, - "Start endpoint watcher on {0} with xDS client {1}", - resourceName, - xdsClient); - xdsClient.watchEndpointData(resourceName, endpointWatcher); - } - - @Override - public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { - EdsConfig config = (EdsConfig) resolvedAddresses.getLoadBalancingPolicyConfig(); - if (config.lrsServerName != null) { - if (!config.lrsServerName.equals("")) { - throw new AssertionError("Can only report load to the same management server"); - } - if (!isReportingLoad) { - xdsClient.reportClientStats(); - isReportingLoad = true; - } - } else { - if (isReportingLoad) { - xdsClient.cancelClientStatsReport(); - isReportingLoad = false; - } - } - // TODO(zddapeng): In handleResolvedAddresses() handle child policy change if any. - } - - @Override - public void handleNameResolutionError(Status error) { - if (!endpointWatcher.endpointsReceived) { - helper.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(error)); - } - } - - @Override - public boolean canHandleEmptyAddressListFromNameResolution() { - return true; - } - - @Override - public void shutdown() { - if (isReportingLoad) { - xdsClient.cancelClientStatsReport(); - isReportingLoad = false; - } - localityStore.reset(); - xdsClient.removeClientStats(clusterName, clusterServiceName); - xdsClient.cancelEndpointDataWatch(resourceName, endpointWatcher); - logger.log( - XdsLogLevel.INFO, - "Cancelled endpoint watcher on {0} with xDS client {1}", - resourceName, - xdsClient); - } - - private final class EndpointWatcherImpl implements EndpointWatcher { - boolean endpointsReceived; - - @Override - public void onEndpointChanged(EndpointUpdate endpointUpdate) { - logger.log(XdsLogLevel.DEBUG, "Received endpoint update: {0}", endpointUpdate); - if (logger.isLoggable(XdsLogLevel.INFO)) { - logger.log( - XdsLogLevel.INFO, - "Received endpoint update from xDS client {0}: cluster_name={1}, {2} localities, " - + "{3} drop categories", - xdsClient, - endpointUpdate.getClusterName(), - endpointUpdate.getLocalityLbEndpointsMap().size(), - endpointUpdate.getDropPolicies().size()); - } - endpointsReceived = true; - List dropOverloads = endpointUpdate.getDropPolicies(); - ImmutableList.Builder dropOverloadsBuilder = ImmutableList.builder(); - for (DropOverload dropOverload : dropOverloads) { - dropOverloadsBuilder.add(dropOverload); - if (dropOverload.getDropsPerMillion() == 1_000_000) { - break; - } - } - localityStore.updateDropPercentage(dropOverloadsBuilder.build()); - - ImmutableMap.Builder localityEndpointsMapping = - new ImmutableMap.Builder<>(); - for (Map.Entry entry - : endpointUpdate.getLocalityLbEndpointsMap().entrySet()) { - int localityWeight = entry.getValue().getLocalityWeight(); - - if (localityWeight != 0) { - localityEndpointsMapping.put(entry.getKey(), entry.getValue()); - } - } - localityStore.updateLocalityStore(localityEndpointsMapping.build()); - } - - @Override - public void onResourceDoesNotExist(String resourceName) { - logger.log(XdsLogLevel.INFO, "Resource {0} is unavailable", resourceName); - if (isReportingLoad) { - xdsClient.cancelClientStatsReport(); - isReportingLoad = false; - } - localityStore.reset(); - helper.updateBalancingState( - TRANSIENT_FAILURE, - new ErrorPicker( - Status.UNAVAILABLE.withDescription( - "Resource " + resourceName + " is unavailable"))); - } - - @Override - public void onError(Status error) { - logger.log( - XdsLogLevel.WARNING, - "Received error from xDS client {0}: {1}: {2}", - xdsClient, - error.getCode(), - error.getDescription()); - if (!endpointsReceived) { - helper.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(error)); - } - } - } - } - } -} diff --git a/xds/src/main/java/io/grpc/xds/LocalityStore.java b/xds/src/main/java/io/grpc/xds/LocalityStore.java deleted file mode 100644 index 4fa507e6a12..00000000000 --- a/xds/src/main/java/io/grpc/xds/LocalityStore.java +++ /dev/null @@ -1,578 +0,0 @@ -/* - * Copyright 2019 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.xds; - -import static com.google.common.base.Preconditions.checkNotNull; -import static io.grpc.ConnectivityState.CONNECTING; -import static io.grpc.ConnectivityState.IDLE; -import static io.grpc.ConnectivityState.READY; -import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.MoreObjects; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import io.grpc.ConnectivityState; -import io.grpc.EquivalentAddressGroup; -import io.grpc.InternalLogId; -import io.grpc.LoadBalancer; -import io.grpc.LoadBalancer.Helper; -import io.grpc.LoadBalancer.PickResult; -import io.grpc.LoadBalancer.PickSubchannelArgs; -import io.grpc.LoadBalancer.ResolvedAddresses; -import io.grpc.LoadBalancer.SubchannelPicker; -import io.grpc.LoadBalancerProvider; -import io.grpc.LoadBalancerRegistry; -import io.grpc.Status; -import io.grpc.SynchronizationContext.ScheduledHandle; -import io.grpc.util.ForwardingLoadBalancerHelper; -import io.grpc.xds.ClientLoadCounter.LoadRecordingSubchannelPicker; -import io.grpc.xds.ClientLoadCounter.MetricsObservingSubchannelPicker; -import io.grpc.xds.ClientLoadCounter.MetricsRecordingListener; -import io.grpc.xds.EnvoyProtoData.DropOverload; -import io.grpc.xds.EnvoyProtoData.LbEndpoint; -import io.grpc.xds.EnvoyProtoData.Locality; -import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints; -import io.grpc.xds.LoadStatsManager.LoadStatsStore; -import io.grpc.xds.OrcaOobUtil.OrcaReportingConfig; -import io.grpc.xds.OrcaOobUtil.OrcaReportingHelperWrapper; -import io.grpc.xds.WeightedRandomPicker.WeightedChildPicker; -import io.grpc.xds.XdsLogger.XdsLogLevel; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import javax.annotation.Nullable; - -/** - * Manages EAG and locality info for a collection of subchannels, not including subchannels - * created by the fallback balancer. - */ -// Must be accessed/run in SynchronizedContext. -interface LocalityStore { - - void reset(); - - void updateLocalityStore(Map localityInfoMap); - - void updateDropPercentage(List dropOverloads); - - void updateOobMetricsReportInterval(long reportIntervalNano); - - @VisibleForTesting - abstract class LocalityStoreFactory { - private static final LocalityStoreFactory DEFAULT_INSTANCE = - new LocalityStoreFactory() { - @Override - LocalityStore newLocalityStore( - InternalLogId logId, - Helper helper, - LoadBalancerRegistry lbRegistry, - LoadStatsStore loadStatsStore) { - return new LocalityStoreImpl(logId, helper, lbRegistry, loadStatsStore); - } - }; - - static LocalityStoreFactory getInstance() { - return DEFAULT_INSTANCE; - } - - abstract LocalityStore newLocalityStore( - InternalLogId logId, - Helper helper, - LoadBalancerRegistry lbRegistry, - LoadStatsStore loadStatsStore); - } - - final class LocalityStoreImpl implements LocalityStore { - private static final String ROUND_ROBIN = "round_robin"; - private static final long DELAYED_DELETION_TIMEOUT_MINUTES = 15L; - - private final XdsLogger logger; - private final Helper helper; - private final LoadBalancerProvider loadBalancerProvider; - private final ThreadSafeRandom random; - private final LoadStatsStore loadStatsStore; - private final OrcaPerRequestUtil orcaPerRequestUtil; - private final OrcaOobUtil orcaOobUtil; - private final PriorityManager priorityManager = new PriorityManager(); - private final Map localityMap = new HashMap<>(); - private List dropOverloads = ImmutableList.of(); - private long metricsReportIntervalNano = -1; - - LocalityStoreImpl( - InternalLogId logId, - Helper helper, - LoadBalancerRegistry lbRegistry, - LoadStatsStore loadStatsStore) { - this( - logId, - helper, - lbRegistry, - ThreadSafeRandom.ThreadSafeRandomImpl.instance, - loadStatsStore, - OrcaPerRequestUtil.getInstance(), - OrcaOobUtil.getInstance()); - } - - @VisibleForTesting - LocalityStoreImpl( - InternalLogId logId, - Helper helper, - LoadBalancerRegistry lbRegistry, - ThreadSafeRandom random, - LoadStatsStore loadStatsStore, - OrcaPerRequestUtil orcaPerRequestUtil, - OrcaOobUtil orcaOobUtil) { - this.helper = checkNotNull(helper, "helper"); - loadBalancerProvider = checkNotNull( - lbRegistry.getProvider(ROUND_ROBIN), - "Unable to find '%s' LoadBalancer", ROUND_ROBIN); - this.random = checkNotNull(random, "random"); - this.loadStatsStore = checkNotNull(loadStatsStore, "loadStatsStore"); - this.orcaPerRequestUtil = checkNotNull(orcaPerRequestUtil, "orcaPerRequestUtil"); - this.orcaOobUtil = checkNotNull(orcaOobUtil, "orcaOobUtil"); - logger = XdsLogger.withLogId(checkNotNull(logId, "logId")); - } - - private final class DroppablePicker extends SubchannelPicker { - - final List dropOverloads; - final SubchannelPicker delegate; - final ThreadSafeRandom random; - final LoadStatsStore loadStatsStore; - - DroppablePicker( - List dropOverloads, SubchannelPicker delegate, - ThreadSafeRandom random, LoadStatsStore loadStatsStore) { - this.dropOverloads = dropOverloads; - this.delegate = delegate; - this.random = random; - this.loadStatsStore = loadStatsStore; - } - - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - for (DropOverload dropOverload : dropOverloads) { - int rand = random.nextInt(1000_000); - if (rand < dropOverload.getDropsPerMillion()) { - logger.log( - XdsLogLevel.INFO, - "Drop request with category: {0}", dropOverload.getCategory()); - loadStatsStore.recordDroppedRequest(dropOverload.getCategory()); - return PickResult.withDrop(Status.UNAVAILABLE.withDescription( - "dropped by loadbalancer: " + dropOverload.toString())); - } - } - return delegate.pickSubchannel(args); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("dropOverloads", dropOverloads) - .add("delegate", delegate) - .toString(); - } - } - - @Override - public void reset() { - for (Locality locality : localityMap.keySet()) { - localityMap.get(locality).shutdown(); - } - localityMap.clear(); - priorityManager.reset(); - } - - @Override - public void updateLocalityStore(final Map localityInfoMap) { - Set newLocalities = localityInfoMap.keySet(); - // TODO: put endPointWeights into attributes for WRR. - for (Locality locality : newLocalities) { - if (localityMap.containsKey(locality)) { - LocalityLbInfo localityLbInfo = localityMap.get(locality); - localityLbInfo.refreshEndpoints(localityInfoMap.get(locality)); - } - } - priorityManager.updateLocalities(localityInfoMap); - for (Locality oldLocality : localityMap.keySet()) { - if (!newLocalities.contains(oldLocality)) { - deactivate(oldLocality); - } - } - } - - @Override - public void updateDropPercentage(List dropOverloads) { - this.dropOverloads = checkNotNull(dropOverloads, "dropOverloads"); - } - - private void deactivate(final Locality locality) { - if (!localityMap.containsKey(locality) || localityMap.get(locality).isDeactivated()) { - return; - } - - final LocalityLbInfo localityLbInfo = localityMap.get(locality); - class DeletionTask implements Runnable { - - @Override - public void run() { - localityLbInfo.shutdown(); - localityMap.remove(locality); - } - - @Override - public String toString() { - return "DeletionTask: locality=" + locality; - } - } - - localityLbInfo.delayedDeletionTimer = helper.getSynchronizationContext().schedule( - new DeletionTask(), DELAYED_DELETION_TIMEOUT_MINUTES, - TimeUnit.MINUTES, helper.getScheduledExecutorService()); - } - - @Override - public void updateOobMetricsReportInterval(long reportIntervalNano) { - metricsReportIntervalNano = reportIntervalNano; - for (LocalityLbInfo lbInfo : localityMap.values()) { - lbInfo.childHelper.updateMetricsReportInterval(reportIntervalNano); - } - } - - @Nullable - private static ConnectivityState aggregateState( - @Nullable ConnectivityState overallState, ConnectivityState childState) { - if (overallState == null) { - return childState; - } - if (overallState == READY || childState == READY) { - return READY; - } - if (overallState == CONNECTING || childState == CONNECTING) { - return CONNECTING; - } - if (overallState == IDLE || childState == IDLE) { - return IDLE; - } - return overallState; - } - - private void updatePicker( - @Nullable ConnectivityState state, List childPickers) { - SubchannelPicker picker; - if (childPickers.isEmpty()) { - if (state == TRANSIENT_FAILURE) { - picker = new ErrorPicker(Status.UNAVAILABLE); // TODO: more details in status - } else { - picker = XdsSubchannelPickers.BUFFER_PICKER; - } - } else { - picker = new WeightedRandomPicker(childPickers); - } - - if (!dropOverloads.isEmpty()) { - picker = new DroppablePicker(dropOverloads, picker, random, loadStatsStore); - } - - if (state != null) { - helper.updateBalancingState(state, picker); - } - } - - /** - * State of a single Locality. - */ - // TODO(zdapeng): rename it to LocalityLbState - private final class LocalityLbInfo { - - final Locality locality; - final LoadBalancer childBalancer; - final ChildHelper childHelper; - @Nullable - private ScheduledHandle delayedDeletionTimer; - - LocalityLbInfo(Locality locality) { - this.locality = checkNotNull(locality, "locality"); - ClientLoadCounter counter = loadStatsStore.addLocality(locality); - childHelper = new ChildHelper(counter); - childBalancer = loadBalancerProvider.newLoadBalancer(childHelper); - } - - void refreshEndpoints(LocalityLbEndpoints localityLbEndpoints) { - final List eags = new ArrayList<>(); - for (LbEndpoint endpoint : localityLbEndpoints.getEndpoints()) { - if (endpoint.isHealthy()) { - eags.add(endpoint.getAddress()); - } - } - // In extreme case handleResolvedAddresses() may trigger updateBalancingState() - // immediately, so execute handleResolvedAddresses() after all the setup in the caller is - // complete. - childHelper.getSynchronizationContext().execute(new Runnable() { - @Override - public void run() { - if (eags.isEmpty() && !childBalancer.canHandleEmptyAddressListFromNameResolution()) { - childBalancer.handleNameResolutionError( - Status.UNAVAILABLE.withDescription( - "Locality " + locality + " has no healthy endpoint")); - } else { - childBalancer.handleResolvedAddresses( - ResolvedAddresses.newBuilder().setAddresses(eags).build()); - } - } - }); - } - - void shutdown() { - if (delayedDeletionTimer != null) { - delayedDeletionTimer.cancel(); - delayedDeletionTimer = null; - } - childBalancer.shutdown(); - loadStatsStore.removeLocality(locality); - logger.log(XdsLogLevel.INFO, "Shut down child balancer for locality {0}", locality); - } - - void reactivate() { - if (delayedDeletionTimer != null) { - delayedDeletionTimer.cancel(); - delayedDeletionTimer = null; - } - } - - boolean isDeactivated() { - return delayedDeletionTimer != null; - } - - class ChildHelper extends ForwardingLoadBalancerHelper { - - private final OrcaReportingHelperWrapper orcaReportingHelperWrapper; - private SubchannelPicker currentChildPicker = XdsSubchannelPickers.BUFFER_PICKER; - private ConnectivityState currentChildState = CONNECTING; - - ChildHelper(final ClientLoadCounter counter) { - Helper delegate = new ForwardingLoadBalancerHelper() { - @Override - protected Helper delegate() { - return helper; - } - - @Override - public void updateBalancingState( - ConnectivityState newState, SubchannelPicker newPicker) { - logger.log( - XdsLogLevel.INFO, - "Update load balancing state for locality {0} to {1}", locality, newState); - currentChildState = newState; - currentChildPicker = - new LoadRecordingSubchannelPicker( - counter, - new MetricsObservingSubchannelPicker(new MetricsRecordingListener(counter), - newPicker, orcaPerRequestUtil)); - - priorityManager.updatePriorityState(priorityManager.getPriority(locality)); - } - - @Override - public String getAuthority() { - //FIXME: This should be a new proposed field of Locality, locality_name - return locality.getSubZone(); - } - }; - - orcaReportingHelperWrapper = - orcaOobUtil.newOrcaReportingHelperWrapper( - delegate, new MetricsRecordingListener(counter)); - if (metricsReportIntervalNano > 0) { - updateMetricsReportInterval(metricsReportIntervalNano); - } - } - - void updateMetricsReportInterval(long intervalNanos) { - orcaReportingHelperWrapper - .setReportingConfig(OrcaReportingConfig.newBuilder() - .setReportInterval(intervalNanos, TimeUnit.NANOSECONDS).build()); - } - - @Override - protected Helper delegate() { - return orcaReportingHelperWrapper.asHelper(); - } - } - } - - private final class PriorityManager { - - private final List> priorityTable = new ArrayList<>(); - private Map localityInfoMap = ImmutableMap.of(); - private int currentPriority = -1; - private ScheduledHandle failOverTimer; - - /** - * Updates the priority ordering of localities with the given collection of localities. - * Recomputes the current ready localities to be used. - */ - void updateLocalities(Map localityInfoMap) { - this.localityInfoMap = localityInfoMap; - priorityTable.clear(); - for (Locality newLocality : localityInfoMap.keySet()) { - int priority = localityInfoMap.get(newLocality).getPriority(); - while (priorityTable.size() <= priority) { - priorityTable.add(new ArrayList()); - } - priorityTable.get(priority).add(newLocality); - } - if (logger.isLoggable(XdsLogLevel.INFO)) { - for (int i = 0; i < priorityTable.size(); i++) { - logger.log( - XdsLogLevel.INFO, - "Priority {0} contains localities: {1}", i, priorityTable.get(i)); - } - } - if (priorityTable.isEmpty()) { - helper.updateBalancingState( - TRANSIENT_FAILURE, - new ErrorPicker(Status.UNAVAILABLE.withDescription("Received 0 locality"))); - return; - } - currentPriority = -1; - failOver(); - } - - /** - * Refreshes the group of localities with the given priority. Recomputes the current ready - * localities to be used. - */ - void updatePriorityState(int priority) { - if (priority == -1 || priority > currentPriority) { - return; - } - List childPickers = new ArrayList<>(); - - ConnectivityState overallState = null; - for (Locality l : priorityTable.get(priority)) { - if (!localityMap.containsKey(l)) { - initLocality(l); - } - LocalityLbInfo localityLbInfo = localityMap.get(l); - localityLbInfo.reactivate(); - ConnectivityState childState = localityLbInfo.childHelper.currentChildState; - SubchannelPicker childPicker = localityLbInfo.childHelper.currentChildPicker; - - overallState = aggregateState(overallState, childState); - - if (READY == childState) { - childPickers.add( - new WeightedChildPicker(localityInfoMap.get(l).getLocalityWeight(), childPicker)); - } - } - logger.log(XdsLogLevel.INFO, "Update priority {0} state to {1}", priority, overallState); - if (priority == currentPriority) { - updatePicker(overallState, childPickers); - if (overallState == READY) { - cancelFailOverTimer(); - } else if (overallState == TRANSIENT_FAILURE) { - cancelFailOverTimer(); - failOver(); - } else if (failOverTimer == null) { - failOver(); - } // else, still connecting and failOverTimer not expired yet, noop - } else if (overallState == READY) { - updatePicker(overallState, childPickers); - cancelFailOverTimer(); - currentPriority = priority; - } - - if (overallState == READY) { - for (int p = priority + 1; p < priorityTable.size(); p++) { - for (Locality xdsLocality : priorityTable.get(p)) { - deactivate(xdsLocality); - } - } - } - } - - int getPriority(Locality locality) { - if (localityInfoMap.containsKey(locality)) { - return localityInfoMap.get(locality).getPriority(); - } - return -1; - } - - void reset() { - cancelFailOverTimer(); - priorityTable.clear(); - localityInfoMap = ImmutableMap.of(); - currentPriority = -1; - } - - private void cancelFailOverTimer() { - if (failOverTimer != null) { - failOverTimer.cancel(); - failOverTimer = null; - } - } - - private void failOver() { - if (currentPriority == priorityTable.size() - 1) { - return; - } - - currentPriority++; - - List localities = priorityTable.get(currentPriority); - boolean initializedBefore = false; - for (Locality locality : localities) { - if (localityMap.containsKey(locality)) { - initializedBefore = true; - localityMap.get(locality).reactivate(); - } else { - initLocality(locality); - } - } - - if (!initializedBefore) { - class FailOverTask implements Runnable { - @Override - public void run() { - logger.log(XdsLogLevel.INFO, "Failing over to priority {0}", currentPriority + 1); - failOverTimer = null; - failOver(); - } - } - - failOverTimer = helper.getSynchronizationContext().schedule( - new FailOverTask(), 10, TimeUnit.SECONDS, helper.getScheduledExecutorService()); - } - - updatePriorityState(currentPriority); - } - - private void initLocality(Locality locality) { - logger.log(XdsLogLevel.INFO, "Create child balancer for locality {0}", locality); - LocalityLbInfo localityLbInfo = new LocalityLbInfo(locality); - localityMap.put(locality, localityLbInfo); - localityLbInfo.refreshEndpoints(localityInfoMap.get(locality)); - } - } - } -} diff --git a/xds/src/main/java/io/grpc/xds/XdsLbPolicies.java b/xds/src/main/java/io/grpc/xds/XdsLbPolicies.java index f0ac6426ee0..f98baf90441 100644 --- a/xds/src/main/java/io/grpc/xds/XdsLbPolicies.java +++ b/xds/src/main/java/io/grpc/xds/XdsLbPolicies.java @@ -23,8 +23,6 @@ final class XdsLbPolicies { static final String PRIORITY_POLICY_NAME = "priority_experimental"; static final String WEIGHTED_TARGET_POLICY_NAME = "weighted_target_experimental"; static final String LRS_POLICY_NAME = "lrs_experimental"; - // TODO(chengyuanzhang): delete routing policy. - static final String XDS_ROUTING_POLICY_NAME = "xds_routing_experimental"; private XdsLbPolicies() {} } diff --git a/xds/src/main/java/io/grpc/xds/XdsRoutingLoadBalancer.java b/xds/src/main/java/io/grpc/xds/XdsRoutingLoadBalancer.java deleted file mode 100644 index 9a18b238862..00000000000 --- a/xds/src/main/java/io/grpc/xds/XdsRoutingLoadBalancer.java +++ /dev/null @@ -1,285 +0,0 @@ -/* - * Copyright 2020 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.xds; - -import static com.google.common.base.Preconditions.checkNotNull; -import static io.grpc.ConnectivityState.CONNECTING; -import static io.grpc.ConnectivityState.IDLE; -import static io.grpc.ConnectivityState.READY; -import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; -import static io.grpc.xds.XdsSubchannelPickers.BUFFER_PICKER; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Sets; -import io.grpc.ConnectivityState; -import io.grpc.InternalLogId; -import io.grpc.LoadBalancer; -import io.grpc.LoadBalancerProvider; -import io.grpc.Metadata; -import io.grpc.Status; -import io.grpc.SynchronizationContext; -import io.grpc.SynchronizationContext.ScheduledHandle; -import io.grpc.internal.ServiceConfigUtil.PolicySelection; -import io.grpc.util.ForwardingLoadBalancerHelper; -import io.grpc.util.GracefulSwitchLoadBalancer; -import io.grpc.xds.XdsLogger.XdsLogLevel; -import io.grpc.xds.XdsRoutingLoadBalancerProvider.Route; -import io.grpc.xds.XdsRoutingLoadBalancerProvider.XdsRoutingConfig; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import javax.annotation.Nullable; - -/** Load balancer for xds_routing policy. */ -final class XdsRoutingLoadBalancer extends LoadBalancer { - - @VisibleForTesting - static final int DELAYED_ACTION_DELETION_TIME_MINUTES = 15; - - private final XdsLogger logger; - private final Helper helper; - private final SynchronizationContext syncContext; - private final ScheduledExecutorService timeService; - private final Map childLbStates = new HashMap<>(); // keyed by action names - - private List routes = ImmutableList.of(); - - XdsRoutingLoadBalancer(Helper helper) { - this.helper = checkNotNull(helper, "helper"); - this.syncContext = checkNotNull(helper.getSynchronizationContext(), "syncContext"); - this.timeService = checkNotNull(helper.getScheduledExecutorService(), "timeService"); - logger = XdsLogger.withLogId( - InternalLogId.allocate("xds-routing-lb", helper.getAuthority())); - logger.log(XdsLogLevel.INFO, "Created"); - } - - @Override - public void handleResolvedAddresses(final ResolvedAddresses resolvedAddresses) { - logger.log(XdsLogLevel.DEBUG, "Received resolution result: {0}", resolvedAddresses); - XdsRoutingConfig xdsRoutingConfig = - (XdsRoutingConfig) resolvedAddresses.getLoadBalancingPolicyConfig(); - Map newActions = xdsRoutingConfig.actions; - for (final String actionName : newActions.keySet()) { - final PolicySelection action = newActions.get(actionName); - if (!childLbStates.containsKey(actionName)) { - childLbStates.put(actionName, new ChildLbState(actionName, action.getProvider())); - } else { - childLbStates.get(actionName).reactivate(action.getProvider()); - } - final LoadBalancer childLb = childLbStates.get(actionName).lb; - syncContext.execute(new Runnable() { - @Override - public void run() { - childLb.handleResolvedAddresses( - resolvedAddresses.toBuilder() - .setLoadBalancingPolicyConfig(action.getConfig()) - .build()); - } - }); - } - this.routes = xdsRoutingConfig.routes; - Set diff = Sets.difference(childLbStates.keySet(), newActions.keySet()); - for (String actionName : diff) { - childLbStates.get(actionName).deactivate(); - } - updateOverallBalancingState(); - } - - @Override - public void handleNameResolutionError(Status error) { - logger.log(XdsLogLevel.WARNING, "Received name resolution error: {0}", error); - boolean gotoTransientFailure = true; - for (ChildLbState state : childLbStates.values()) { - if (!state.deactivated) { - gotoTransientFailure = false; - state.lb.handleNameResolutionError(error); - } - } - if (gotoTransientFailure) { - helper.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(error)); - } - } - - @Override - public void shutdown() { - logger.log(XdsLogLevel.INFO, "Shutdown"); - for (ChildLbState state : childLbStates.values()) { - state.tearDown(); - } - } - - @Override - public boolean canHandleEmptyAddressListFromNameResolution() { - return true; - } - - private void updateOverallBalancingState() { - ConnectivityState overallState = null; - // Use LinkedHashMap to preserve the order of routes. - Map routePickers = new LinkedHashMap<>(); - for (Route route : routes) { - ChildLbState state = childLbStates.get(route.getActionName()); - routePickers.put(route.getRouteMatch(), state.currentPicker); - overallState = aggregateState(overallState, state.currentState); - } - if (overallState != null) { - SubchannelPicker picker = new RouteMatchingSubchannelPicker(routePickers); - helper.updateBalancingState(overallState, picker); - } - } - - @VisibleForTesting - @Nullable - static ConnectivityState aggregateState( - @Nullable ConnectivityState overallState, ConnectivityState childState) { - if (overallState == null) { - return childState; - } - if (overallState == READY || childState == READY) { - return READY; - } - if (overallState == CONNECTING || childState == CONNECTING) { - return CONNECTING; - } - if (overallState == IDLE || childState == IDLE) { - return IDLE; - } - return overallState; - } - - private final class ChildLbState { - private final String name; - private final GracefulSwitchLoadBalancer lb; - private LoadBalancerProvider policyProvider; - private ConnectivityState currentState = CONNECTING; - private SubchannelPicker currentPicker = BUFFER_PICKER; - private boolean deactivated; - @Nullable - ScheduledHandle deletionTimer; - - private ChildLbState(String name, LoadBalancerProvider policyProvider) { - this.name = name; - this.policyProvider = policyProvider; - lb = new GracefulSwitchLoadBalancer(new RouteHelper()); - lb.switchTo(policyProvider); - } - - void deactivate() { - if (deactivated) { - return; - } - - class DeletionTask implements Runnable { - @Override - public void run() { - tearDown(); - childLbStates.remove(name); - } - } - - deletionTimer = - syncContext.schedule( - new DeletionTask(), - DELAYED_ACTION_DELETION_TIME_MINUTES, - TimeUnit.MINUTES, - timeService); - deactivated = true; - logger.log(XdsLogLevel.DEBUG, "Route action {0} deactivated", name); - } - - void reactivate(LoadBalancerProvider policyProvider) { - if (deletionTimer != null && deletionTimer.isPending()) { - deletionTimer.cancel(); - deactivated = false; - logger.log(XdsLogLevel.DEBUG, "Route action {0} reactivated", name); - } - if (!this.policyProvider.getPolicyName().equals(policyProvider.getPolicyName())) { - logger.log( - XdsLogLevel.DEBUG, - "Action {0} switching policy from {1} to {2}", - name, this.policyProvider.getPolicyName(), policyProvider.getPolicyName()); - lb.switchTo(policyProvider); - this.policyProvider = policyProvider; - } - } - - void tearDown() { - deactivated = true; - if (deletionTimer != null && deletionTimer.isPending()) { - deletionTimer.cancel(); - } - lb.shutdown(); - logger.log(XdsLogLevel.DEBUG, "Route action {0} deleted", name); - } - - private final class RouteHelper extends ForwardingLoadBalancerHelper { - - @Override - public void updateBalancingState(ConnectivityState newState, SubchannelPicker newPicker) { - currentState = newState; - currentPicker = newPicker; - if (!deactivated) { - updateOverallBalancingState(); - } - } - - @Override - protected Helper delegate() { - return helper; - } - } - } - - @VisibleForTesting - static final class RouteMatchingSubchannelPicker extends SubchannelPicker { - - @VisibleForTesting - final Map routePickers; - - RouteMatchingSubchannelPicker(Map routePickers) { - this.routePickers = routePickers; - } - - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - // Index ASCII headers by keys. - Map> asciiHeaders = new HashMap<>(); - Metadata headers = args.getHeaders(); - for (String headerName : headers.keys()) { - if (headerName.endsWith(Metadata.BINARY_HEADER_SUFFIX)) { - continue; - } - Metadata.Key key = Metadata.Key.of(headerName, Metadata.ASCII_STRING_MARSHALLER); - asciiHeaders.put(headerName, headers.getAll(key)); - } - for (Map.Entry entry : routePickers.entrySet()) { - RouteMatch routeMatch = entry.getKey(); - if (routeMatch.matches( - "/" + args.getMethodDescriptor().getFullMethodName(), asciiHeaders)) { - return entry.getValue().pickSubchannel(args); - } - } - return PickResult.withError(Status.UNAVAILABLE.withDescription("no matching route found")); - } - } -} diff --git a/xds/src/main/java/io/grpc/xds/XdsRoutingLoadBalancerProvider.java b/xds/src/main/java/io/grpc/xds/XdsRoutingLoadBalancerProvider.java deleted file mode 100644 index ee49050bdac..00000000000 --- a/xds/src/main/java/io/grpc/xds/XdsRoutingLoadBalancerProvider.java +++ /dev/null @@ -1,348 +0,0 @@ -/* - * Copyright 2020 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.xds; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.MoreObjects; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.re2j.Pattern; -import com.google.re2j.PatternSyntaxException; -import io.grpc.Internal; -import io.grpc.LoadBalancer; -import io.grpc.LoadBalancer.Helper; -import io.grpc.LoadBalancerProvider; -import io.grpc.LoadBalancerRegistry; -import io.grpc.NameResolver.ConfigOrError; -import io.grpc.Status; -import io.grpc.internal.JsonUtil; -import io.grpc.internal.ServiceConfigUtil; -import io.grpc.internal.ServiceConfigUtil.LbConfig; -import io.grpc.internal.ServiceConfigUtil.PolicySelection; -import io.grpc.xds.RouteMatch.FractionMatcher; -import io.grpc.xds.RouteMatch.HeaderMatcher; -import io.grpc.xds.RouteMatch.PathMatcher; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import javax.annotation.Nullable; - -/** - * The provider for the xds_routing balancing policy. This class should not be directly referenced - * in code. The policy should be accessed through {@link LoadBalancerRegistry#getProvider} with the - * name "xds_routing_experimental". - */ -@Internal -public final class XdsRoutingLoadBalancerProvider extends LoadBalancerProvider { - - @Nullable - private final LoadBalancerRegistry lbRegistry; - - // We can not call this(LoadBalancerRegistry.getDefaultRegistry()), because it will get stuck - // recursively loading LoadBalancerRegistry and XdsRoutingLoadBalancerProvider. - public XdsRoutingLoadBalancerProvider() { - this(null); - } - - @VisibleForTesting - XdsRoutingLoadBalancerProvider(@Nullable LoadBalancerRegistry lbRegistry) { - this.lbRegistry = lbRegistry; - } - - @Override - public boolean isAvailable() { - return true; - } - - @Override - public int getPriority() { - return 5; - } - - @Override - public String getPolicyName() { - return XdsLbPolicies.XDS_ROUTING_POLICY_NAME; - } - - @Override - public LoadBalancer newLoadBalancer(Helper helper) { - return new XdsRoutingLoadBalancer(helper); - } - - @Override - public ConfigOrError parseLoadBalancingPolicyConfig(Map rawConfig) { - try { - Map actions = JsonUtil.getObject(rawConfig, "action"); - if (actions == null || actions.isEmpty()) { - return ConfigOrError.fromError(Status.INTERNAL.withDescription( - "No actions provided for xds_routing LB policy: " + rawConfig)); - } - Map parsedActions = new LinkedHashMap<>(); - for (String name : actions.keySet()) { - Map rawAction = JsonUtil.getObject(actions, name); - if (rawAction == null) { - return ConfigOrError.fromError(Status.INTERNAL.withDescription( - "No config for action " + name + " in xds_routing LB policy: " + rawConfig)); - } - PolicySelection parsedAction = - parseAction( - rawAction, - this.lbRegistry == null - ? LoadBalancerRegistry.getDefaultRegistry() : this.lbRegistry); - parsedActions.put(name, parsedAction); - } - - List parsedRoutes = new ArrayList<>(); - List> rawRoutes = JsonUtil.getListOfObjects(rawConfig, "route"); - if (rawRoutes == null || rawRoutes.isEmpty()) { - return ConfigOrError.fromError(Status.INTERNAL.withDescription( - "No routes provided for xds_routing LB policy: " + rawConfig)); - } - for (Map rawRoute: rawRoutes) { - Route route = parseRoute(rawRoute); - if (!parsedActions.containsKey(route.getActionName())) { - return ConfigOrError.fromError(Status.INTERNAL.withDescription( - "No action defined for route " + route + " in xds_routing LB policy: " + rawConfig)); - } - parsedRoutes.add(route); - } - return ConfigOrError.fromConfig(new XdsRoutingConfig(parsedRoutes, parsedActions)); - } catch (RuntimeException e) { - return ConfigOrError.fromError( - Status.fromThrowable(e).withDescription( - "Failed to parse xds_routing LB config: " + rawConfig)); - } - } - - private static PolicySelection parseAction( - Map rawAction, LoadBalancerRegistry registry) { - List childConfigCandidates = ServiceConfigUtil.unwrapLoadBalancingConfigList( - JsonUtil.getListOfObjects(rawAction, "childPolicy")); - if (childConfigCandidates == null || childConfigCandidates.isEmpty()) { - throw new RuntimeException("childPolicy not specified"); - } - ConfigOrError selectedConfigOrError = - ServiceConfigUtil.selectLbPolicyFromList(childConfigCandidates, registry); - if (selectedConfigOrError.getError() != null) { - throw selectedConfigOrError.getError().asRuntimeException(); - } - return (PolicySelection) selectedConfigOrError.getConfig(); - } - - private static Route parseRoute(Map rawRoute) { - try { - String pathExact = JsonUtil.getString(rawRoute, "path"); - String pathPrefix = JsonUtil.getString(rawRoute, "prefix"); - Pattern pathRegex = null; - String rawPathRegex = JsonUtil.getString(rawRoute, "regex"); - if (rawPathRegex != null) { - try { - pathRegex = Pattern.compile(rawPathRegex); - } catch (PatternSyntaxException e) { - throw new RuntimeException(e); - } - } - if (!isOneOf(pathExact, pathPrefix, pathRegex)) { - throw new RuntimeException("must specify exactly one patch match type"); - } - PathMatcher pathMatcher = new PathMatcher(pathExact, pathPrefix, pathRegex); - - List headers = new ArrayList<>(); - List> rawHeaders = JsonUtil.getListOfObjects(rawRoute, "headers"); - if (rawHeaders != null) { - for (Map rawHeader : rawHeaders) { - HeaderMatcher headerMatcher = parseHeaderMatcher(rawHeader); - headers.add(headerMatcher); - } - } - - FractionMatcher matchFraction = null; - Map rawFraction = JsonUtil.getObject(rawRoute, "matchFraction"); - if (rawFraction != null) { - matchFraction = parseFractionMatcher(rawFraction); - } - - String actionName = JsonUtil.getString(rawRoute, "action"); - if (actionName == null) { - throw new RuntimeException("action name not specified"); - } - return new Route(new RouteMatch(pathMatcher, headers, matchFraction), actionName); - } catch (RuntimeException e) { - throw new RuntimeException("Failed to parse Route: " + e.getMessage()); - } - } - - private static HeaderMatcher parseHeaderMatcher(Map rawHeaderMatcher) { - try { - String name = JsonUtil.getString(rawHeaderMatcher, "name"); - if (name == null) { - throw new RuntimeException("header name not specified"); - } - String exactMatch = JsonUtil.getString(rawHeaderMatcher, "exactMatch"); - Pattern regexMatch = null; - String rawRegex = JsonUtil.getString(rawHeaderMatcher, "regexMatch"); - if (rawRegex != null) { - try { - regexMatch = Pattern.compile(rawRegex); - } catch (PatternSyntaxException e) { - throw new RuntimeException(e); - } - } - Map rawRangeMatch = JsonUtil.getObject(rawHeaderMatcher, "rangeMatch"); - HeaderMatcher.Range rangeMatch = - rawRangeMatch == null ? null : parseHeaderRange(rawRangeMatch); - Boolean presentMatch = JsonUtil.getBoolean(rawHeaderMatcher, "presentMatch"); - String prefixMatch = JsonUtil.getString(rawHeaderMatcher, "prefixMatch"); - String suffixMatch = JsonUtil.getString(rawHeaderMatcher, "suffixMatch"); - if (!isOneOf(exactMatch, regexMatch, rangeMatch, presentMatch, prefixMatch, suffixMatch)) { - throw new RuntimeException("must specify exactly one match type"); - } - Boolean inverted = JsonUtil.getBoolean(rawHeaderMatcher, "invertMatch"); - return new HeaderMatcher( - name, exactMatch, regexMatch, rangeMatch, presentMatch, prefixMatch, suffixMatch, - inverted == null ? false : inverted); - } catch (RuntimeException e) { - throw new RuntimeException("Failed to parse HeaderMatcher: " + e.getMessage()); - } - } - - private static boolean isOneOf(Object... objects) { - int count = 0; - for (Object o : objects) { - if (o != null) { - count++; - } - } - return count == 1; - } - - private static HeaderMatcher.Range parseHeaderRange(Map rawRange) { - try { - Long start = JsonUtil.getNumberAsLong(rawRange, "start"); - if (start == null) { - throw new RuntimeException("start not specified"); - } - Long end = JsonUtil.getNumberAsLong(rawRange, "end"); - if (end == null) { - throw new RuntimeException("end not specified"); - } - return new HeaderMatcher.Range(start, end); - } catch (RuntimeException e) { - throw new RuntimeException("Failed to parse Range: " + e.getMessage()); - } - } - - private static FractionMatcher parseFractionMatcher(Map rawFraction) { - try { - Integer numerator = JsonUtil.getNumberAsInteger(rawFraction, "numerator"); - if (numerator == null) { - throw new RuntimeException("numerator not specified"); - } - Integer denominator = JsonUtil.getNumberAsInteger(rawFraction, "denominator"); - if (denominator == null) { - throw new RuntimeException("denominator not specified"); - } - return new FractionMatcher(numerator, denominator); - } catch (RuntimeException e) { - throw new RuntimeException("Failed to parse Fraction: " + e.getMessage()); - } - } - - static final class XdsRoutingConfig { - - final List routes; - final Map actions; - - @VisibleForTesting - XdsRoutingConfig(List routes, Map actions) { - this.routes = ImmutableList.copyOf(routes); - this.actions = ImmutableMap.copyOf(actions); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - XdsRoutingConfig that = (XdsRoutingConfig) o; - return Objects.equals(routes, that.routes) - && Objects.equals(actions, that.actions); - } - - @Override - public int hashCode() { - return Objects.hash(routes, actions); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("routes", routes) - .add("actions", actions) - .toString(); - } - } - - static final class Route { - private final RouteMatch routeMatch; - private final String actionName; - - Route(RouteMatch routeMatch, String actionName) { - this.routeMatch = routeMatch; - this.actionName = actionName; - } - - String getActionName() { - return actionName; - } - - RouteMatch getRouteMatch() { - return routeMatch; - } - - @Override - public int hashCode() { - return Objects.hash(routeMatch, actionName); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Route that = (Route) o; - return Objects.equals(actionName, that.actionName) - && Objects.equals(routeMatch, that.routeMatch); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("routeMatch", routeMatch) - .add("actionName", actionName) - .toString(); - } - } -} diff --git a/xds/src/main/resources/META-INF/services/io.grpc.LoadBalancerProvider b/xds/src/main/resources/META-INF/services/io.grpc.LoadBalancerProvider index ff774497bb4..c9afd7b4c47 100644 --- a/xds/src/main/resources/META-INF/services/io.grpc.LoadBalancerProvider +++ b/xds/src/main/resources/META-INF/services/io.grpc.LoadBalancerProvider @@ -3,5 +3,4 @@ io.grpc.xds.EdsLoadBalancerProvider io.grpc.xds.PriorityLoadBalancerProvider io.grpc.xds.WeightedTargetLoadBalancerProvider io.grpc.xds.LrsLoadBalancerProvider -io.grpc.xds.XdsRoutingLoadBalancerProvider io.grpc.xds.ClusterManagerLoadBalancerProvider diff --git a/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java deleted file mode 100644 index bc83a873863..00000000000 --- a/xds/src/test/java/io/grpc/xds/EdsLoadBalancerTest.java +++ /dev/null @@ -1,866 +0,0 @@ -/* - * Copyright 2019 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.xds; - -import static com.google.common.truth.Truth.assertThat; -import static io.envoyproxy.envoy.api.v2.core.HealthStatus.HEALTHY; -import static io.grpc.ConnectivityState.CONNECTING; -import static io.grpc.ConnectivityState.READY; -import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; -import static io.grpc.xds.XdsClientTestHelper.buildClusterLoadAssignmentV2; -import static io.grpc.xds.XdsClientTestHelper.buildDiscoveryResponseV2; -import static io.grpc.xds.XdsClientTestHelper.buildDropOverloadV2; -import static io.grpc.xds.XdsClientTestHelper.buildLbEndpointV2; -import static io.grpc.xds.XdsClientTestHelper.buildLocalityLbEndpointsV2; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.argThat; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.atLeastOnce; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; -import com.google.protobuf.Any; -import io.envoyproxy.envoy.api.v2.ClusterLoadAssignment; -import io.envoyproxy.envoy.api.v2.ClusterLoadAssignment.Policy.DropOverload; -import io.envoyproxy.envoy.api.v2.DiscoveryRequest; -import io.envoyproxy.envoy.api.v2.DiscoveryResponse; -import io.envoyproxy.envoy.api.v2.endpoint.LbEndpoint; -import io.envoyproxy.envoy.api.v2.endpoint.LocalityLbEndpoints; -import io.envoyproxy.envoy.service.discovery.v2.AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceImplBase; -import io.grpc.Attributes; -import io.grpc.ConnectivityState; -import io.grpc.EquivalentAddressGroup; -import io.grpc.InternalLogId; -import io.grpc.LoadBalancer; -import io.grpc.LoadBalancer.Helper; -import io.grpc.LoadBalancer.PickResult; -import io.grpc.LoadBalancer.PickSubchannelArgs; -import io.grpc.LoadBalancer.ResolvedAddresses; -import io.grpc.LoadBalancer.Subchannel; -import io.grpc.LoadBalancer.SubchannelPicker; -import io.grpc.LoadBalancerProvider; -import io.grpc.LoadBalancerRegistry; -import io.grpc.ManagedChannel; -import io.grpc.Status; -import io.grpc.Status.Code; -import io.grpc.SynchronizationContext; -import io.grpc.inprocess.InProcessChannelBuilder; -import io.grpc.inprocess.InProcessServerBuilder; -import io.grpc.internal.BackoffPolicy; -import io.grpc.internal.FakeClock; -import io.grpc.internal.ObjectPool; -import io.grpc.internal.ServiceConfigUtil.PolicySelection; -import io.grpc.stub.StreamObserver; -import io.grpc.testing.GrpcCleanupRule; -import io.grpc.xds.Bootstrapper.BootstrapInfo; -import io.grpc.xds.Bootstrapper.ChannelCreds; -import io.grpc.xds.Bootstrapper.ServerInfo; -import io.grpc.xds.EdsLoadBalancerProvider.EdsConfig; -import io.grpc.xds.EnvoyProtoData.Node; -import io.grpc.xds.LoadStatsManager.LoadStatsStore; -import io.grpc.xds.LocalityStore.LocalityStoreFactory; -import io.grpc.xds.XdsClient.EndpointUpdate; -import io.grpc.xds.XdsClient.XdsChannel; -import java.net.InetSocketAddress; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import javax.annotation.Nullable; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameter; -import org.junit.runners.Parameterized.Parameters; -import org.mockito.ArgumentCaptor; -import org.mockito.ArgumentMatcher; -import org.mockito.Captor; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; - -/** - * Tests for {@link EdsLoadBalancer}. - */ -@RunWith(Parameterized.class) -public class EdsLoadBalancerTest { - - private static final String CLUSTER_NAME = "eds-lb-test.example.com"; - private static final String SERVICE_AUTHORITY = "test.authority.example.com"; - - @Rule - public final MockitoRule mockitoRule = MockitoJUnit.rule(); - @Rule - public final GrpcCleanupRule cleanupRule = new GrpcCleanupRule(); - - private final SynchronizationContext syncContext = new SynchronizationContext( - new Thread.UncaughtExceptionHandler() { - @Override - public void uncaughtException(Thread t, Throwable e) { - throw new AssertionError(e); - } - }); - private final FakeClock fakeClock = new FakeClock(); - - private final LoadBalancerRegistry lbRegistry = new LoadBalancerRegistry(); - - // Child helpers keyed by locality names. - private final Map childHelpers = new HashMap<>(); - // Child balancers keyed by locality names. - private final Map childBalancers = new HashMap<>(); - private final XdsChannelFactory channelFactory = new XdsChannelFactory() { - @Override - XdsChannel createChannel(List servers) { - assertThat(Iterables.getOnlyElement(servers).getServerUri()) - .isEqualTo("trafficdirector.googleapis.com"); - return new XdsChannel(channel, false); - } - }; - - private final PolicySelection fakeLocalityPickingPolicy = - new PolicySelection(mock(LoadBalancerProvider.class), null); - private final PolicySelection fakeEndpointPickingPolicy = - new PolicySelection(mock(LoadBalancerProvider.class), new Object()); - - @Mock - private Helper helper; - @Mock - private Bootstrapper bootstrapper; - @Captor - ArgumentCaptor connectivityStateCaptor; - @Captor - ArgumentCaptor pickerCaptor; - - private LoadBalancer edsLb; - // Simulating a CDS to EDS flow, otherwise EDS only. - @Parameter - public boolean isFullFlow; - private ManagedChannel channel; - // Response observer on server side. - private StreamObserver responseObserver; - @Nullable - private FakeXdsClientPool xdsClientPoolFromResolveAddresses; - private LocalityStoreFactory localityStoreFactory = LocalityStoreFactory.getInstance(); - private int versionIno; - private int nonce; - - @Parameters - public static Collection isFullFlow() { - return ImmutableList.of(false, true ); - } - - @Before - public void setUp() throws Exception { - doReturn(SERVICE_AUTHORITY).when(helper).getAuthority(); - doReturn(syncContext).when(helper).getSynchronizationContext(); - doReturn(fakeClock.getScheduledExecutorService()).when(helper).getScheduledExecutorService(); - - // Register a fake round robin balancer provider. - lbRegistry.register(new LoadBalancerProvider() { - @Override - public boolean isAvailable() { - return true; - } - - @Override - public int getPriority() { - return 5; - } - - @Override - public String getPolicyName() { - return "round_robin"; - } - - @Override - public LoadBalancer newLoadBalancer(Helper helper) { - String localityName = helper.getAuthority(); - childHelpers.put(localityName, helper); - LoadBalancer balancer = mock(LoadBalancer.class); - childBalancers.put(localityName, balancer); - return balancer; - } - }); - - AggregatedDiscoveryServiceImplBase serviceImpl = new AggregatedDiscoveryServiceImplBase() { - @Override - public StreamObserver streamAggregatedResources( - final StreamObserver responseObserver) { - EdsLoadBalancerTest.this.responseObserver = responseObserver; - @SuppressWarnings("unchecked") - StreamObserver requestObserver = mock(StreamObserver.class); - return requestObserver; - } - }; - String serverName = InProcessServerBuilder.generateName(); - cleanupRule.register( - InProcessServerBuilder - .forName(serverName) - .directExecutor() - .addService(serviceImpl) - .build() - .start()); - channel = cleanupRule.register( - InProcessChannelBuilder - .forName(serverName) - .directExecutor() - .build()); - final List serverList = ImmutableList.of( - new ServerInfo("trafficdirector.googleapis.com", ImmutableList.of(), null)); - Node node = Node.newBuilder().build(); - BootstrapInfo bootstrapInfo = new BootstrapInfo(serverList, node, null); - doReturn(bootstrapInfo).when(bootstrapper).readBootstrap(); - - if (isFullFlow) { - xdsClientPoolFromResolveAddresses = new FakeXdsClientPool( - new XdsClientImpl( - SERVICE_AUTHORITY, - new XdsChannel(channel, /* useProtocolV3= */ false), - node, - syncContext, - fakeClock.getScheduledExecutorService(), - mock(BackoffPolicy.Provider.class), - fakeClock.getStopwatchSupplier())); - } - - edsLb = - new EdsLoadBalancer(helper, lbRegistry, localityStoreFactory, bootstrapper, channelFactory); - } - - @After - public void tearDown() { - edsLb.shutdown(); - - for (LoadBalancer childBalancer : childBalancers.values()) { - verify(childBalancer).shutdown(); - } - - if (isFullFlow) { - assertThat(xdsClientPoolFromResolveAddresses.timesGetObjectCalled) - .isEqualTo(xdsClientPoolFromResolveAddresses.timesReturnObjectCalled); - - // Just for cleaning up the test. - xdsClientPoolFromResolveAddresses.xdsClient.shutdown(); - } - - assertThat(channel.isShutdown()).isTrue(); - } - - @Test - public void handleNameResolutionErrorBeforeAndAfterEdsWorkding() { - deliverResolvedAddresses(null, null, fakeLocalityPickingPolicy, fakeEndpointPickingPolicy); - - // handleResolutionError() before receiving any endpoint update. - edsLb.handleNameResolutionError(Status.DATA_LOSS.withDescription("fake status")); - verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); - - // Endpoint update received. - ClusterLoadAssignment clusterLoadAssignment = - buildClusterLoadAssignmentV2(CLUSTER_NAME, - ImmutableList.of( - buildLocalityLbEndpointsV2("region1", "zone1", "subzone1", - ImmutableList.of( - buildLbEndpointV2("192.168.0.1", 8080, HEALTHY, 2)), - 1, 0)), - ImmutableList.of(buildDropOverloadV2("throttle", 1000))); - deliverClusterLoadAssignments(clusterLoadAssignment); - - // handleResolutionError() after receiving endpoint update. - edsLb.handleNameResolutionError(Status.DATA_LOSS.withDescription("fake status")); - // No more TRANSIENT_FAILURE. - verify(helper, times(1)).updateBalancingState( - eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); - } - - @Test - public void handleEdsServiceNameChange() { - assertThat(childHelpers).isEmpty(); - - deliverResolvedAddresses("edsServiceName1", null, fakeLocalityPickingPolicy, - fakeEndpointPickingPolicy); - ClusterLoadAssignment clusterLoadAssignment = - buildClusterLoadAssignmentV2("edsServiceName1", - ImmutableList.of( - buildLocalityLbEndpointsV2("region1", "zone1", "subzone1", - ImmutableList.of( - buildLbEndpointV2("192.168.0.1", 8080, HEALTHY, 2)), - 1, 0)), - ImmutableList.of()); - deliverClusterLoadAssignments(clusterLoadAssignment); - assertThat(childHelpers).hasSize(1); - Helper childHelper1 = childHelpers.get("subzone1"); - LoadBalancer childBalancer1 = childBalancers.get("subzone1"); - verify(childBalancer1).handleResolvedAddresses( - argThat(RoundRobinBackendsMatcher.builder().addHostAndPort("192.168.0.1", 8080).build())); - - childHelper1.updateBalancingState(CONNECTING, mock(SubchannelPicker.class)); - assertLatestConnectivityState(CONNECTING); - - // Change edsServicename to edsServiceName2. - deliverResolvedAddresses("edsServiceName2", null, fakeLocalityPickingPolicy, - fakeEndpointPickingPolicy); - // The old balancer was not READY, so it will be shutdown immediately. - verify(childBalancer1).shutdown(); - - clusterLoadAssignment = - buildClusterLoadAssignmentV2("edsServiceName2", - ImmutableList.of( - buildLocalityLbEndpointsV2("region2", "zone2", "subzone2", - ImmutableList.of( - buildLbEndpointV2("192.168.0.2", 8080, HEALTHY, 2)), - 1, 0)), - ImmutableList.of()); - deliverClusterLoadAssignments(clusterLoadAssignment); - assertThat(childHelpers).hasSize(2); - Helper childHelper2 = childHelpers.get("subzone2"); - LoadBalancer childBalancer2 = childBalancers.get("subzone2"); - verify(childBalancer2).handleResolvedAddresses( - argThat(RoundRobinBackendsMatcher.builder().addHostAndPort("192.168.0.2", 8080).build())); - - final Subchannel subchannel2 = mock(Subchannel.class); - SubchannelPicker picker2 = new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withSubchannel(subchannel2); - } - }; - childHelper2.updateBalancingState(READY, picker2); - assertLatestSubchannelPicker(subchannel2); - - // Change edsServiceName to edsServiceName3. - deliverResolvedAddresses("edsServiceName3", null, fakeLocalityPickingPolicy, - fakeEndpointPickingPolicy); - clusterLoadAssignment = - buildClusterLoadAssignmentV2("edsServiceName3", - ImmutableList.of( - buildLocalityLbEndpointsV2("region3", "zone3", "subzone3", - ImmutableList.of( - buildLbEndpointV2("192.168.0.3", 8080, HEALTHY, 2)), - 1, 0)), - ImmutableList.of()); - deliverClusterLoadAssignments(clusterLoadAssignment); - - assertThat(childHelpers).hasSize(3); - Helper childHelper3 = childHelpers.get("subzone3"); - LoadBalancer childBalancer3 = childBalancers.get("subzone3"); - - childHelper3.updateBalancingState(CONNECTING, mock(SubchannelPicker.class)); - // The new balancer is not READY while the old one is still READY. - verify(childBalancer2, never()).shutdown(); - assertLatestSubchannelPicker(subchannel2); - - childHelper2.updateBalancingState(CONNECTING, mock(SubchannelPicker.class)); - // The old balancer becomes not READY, so the new balancer will update picker immediately. - verify(childBalancer2).shutdown(); - assertLatestConnectivityState(CONNECTING); - - // Change edsServiceName to edsServiceName4. - deliverResolvedAddresses("edsServiceName4", null, fakeLocalityPickingPolicy, - fakeEndpointPickingPolicy); - verify(childBalancer3).shutdown(); - - clusterLoadAssignment = - buildClusterLoadAssignmentV2("edsServiceName4", - ImmutableList.of( - buildLocalityLbEndpointsV2("region4", "zone4", "subzone4", - ImmutableList.of( - buildLbEndpointV2("192.168.0.4", 8080, HEALTHY, 2)), - 1, 0)), - ImmutableList.of()); - deliverClusterLoadAssignments(clusterLoadAssignment); - - assertThat(childHelpers).hasSize(4); - Helper childHelper4 = childHelpers.get("subzone4"); - LoadBalancer childBalancer4 = childBalancers.get("subzone4"); - - final Subchannel subchannel4 = mock(Subchannel.class); - SubchannelPicker picker4 = new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withSubchannel(subchannel4); - } - }; - childHelper4.updateBalancingState(READY, picker4); - assertLatestSubchannelPicker(subchannel4); - - // Change edsServiceName to edsServiceName5. - deliverResolvedAddresses("edsServiceName5", null, fakeLocalityPickingPolicy, - fakeEndpointPickingPolicy); - clusterLoadAssignment = - buildClusterLoadAssignmentV2("edsServiceName5", - ImmutableList.of( - buildLocalityLbEndpointsV2("region5", "zone5", "subzone5", - ImmutableList.of( - buildLbEndpointV2("192.168.0.5", 8080, HEALTHY, 2)), - 1, 0)), - ImmutableList.of()); - deliverClusterLoadAssignments(clusterLoadAssignment); - - assertThat(childHelpers).hasSize(5); - Helper childHelper5 = childHelpers.get("subzone5"); - LoadBalancer childBalancer5 = childBalancers.get("subzone5"); - childHelper5.updateBalancingState(CONNECTING, mock(SubchannelPicker.class)); - // The old balancer was READY, so the new balancer will gracefully switch and not update - // non-READY picker. - verify(childBalancer4, never()).shutdown(); - assertLatestSubchannelPicker(subchannel4); - - final Subchannel subchannel5 = mock(Subchannel.class); - SubchannelPicker picker5 = new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withSubchannel(subchannel5); - } - }; - childHelper5.updateBalancingState(READY, picker5); - verify(childBalancer4).shutdown(); - assertLatestSubchannelPicker(subchannel5); - verify(childBalancer5, never()).shutdown(); - } - - @Test - public void edsResourceUpdate_allDrop() { - deliverResolvedAddresses(null, null, fakeLocalityPickingPolicy, fakeEndpointPickingPolicy); - - ClusterLoadAssignment clusterLoadAssignment = buildClusterLoadAssignmentV2( - CLUSTER_NAME, - ImmutableList.of( - buildLocalityLbEndpointsV2("region1", "zone1", "subzone1", - ImmutableList.of( - buildLbEndpointV2("192.168.0.1", 8080, HEALTHY, 2)), - 1, 0)), - ImmutableList.of()); - deliverClusterLoadAssignments(clusterLoadAssignment); - - assertThat(childBalancers).hasSize(1); - verify(childBalancers.get("subzone1")).handleResolvedAddresses( - argThat(RoundRobinBackendsMatcher.builder().addHostAndPort("192.168.0.1", 8080).build())); - assertThat(childHelpers).hasSize(1); - Helper childHelper = childHelpers.get("subzone1"); - - final Subchannel subchannel = mock(Subchannel.class); - SubchannelPicker picker = new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withSubchannel(subchannel); - } - }; - childHelper.updateBalancingState(READY, picker); - assertLatestSubchannelPicker(subchannel); - - clusterLoadAssignment = buildClusterLoadAssignmentV2( - CLUSTER_NAME, - ImmutableList.of( - buildLocalityLbEndpointsV2("region1", "zone1", "subzone1", - ImmutableList.of( - buildLbEndpointV2("192.168.0.1", 8080, HEALTHY, 2)), - 1, 0)), - ImmutableList.of( - buildDropOverloadV2("cat_1", 3), - buildDropOverloadV2("cat_2", 1_000_001), - buildDropOverloadV2("cat_3", 4))); - deliverClusterLoadAssignments(clusterLoadAssignment); - - verify(helper, atLeastOnce()).updateBalancingState(eq(READY), pickerCaptor.capture()); - SubchannelPicker pickerExpectedDropAll = pickerCaptor.getValue(); - assertThat(pickerExpectedDropAll.pickSubchannel(mock(PickSubchannelArgs.class)).isDrop()) - .isTrue(); - } - - @Test - public void edsResourceUpdate_localityAssignmentChange() { - deliverResolvedAddresses(null, null, fakeLocalityPickingPolicy, fakeEndpointPickingPolicy); - - LbEndpoint endpoint11 = buildLbEndpointV2("addr11.example.com", 8011, HEALTHY, 11); - LbEndpoint endpoint12 = buildLbEndpointV2("addr12.example.com", 8012, HEALTHY, 12); - LocalityLbEndpoints localityLbEndpoints1 = buildLocalityLbEndpointsV2( - "region1", "zone1", "subzone1", - ImmutableList.of(endpoint11, endpoint12), - 1, - 0); - - LbEndpoint endpoint21 = buildLbEndpointV2("addr21.example.com", 8021, HEALTHY, 21); - LbEndpoint endpoint22 = buildLbEndpointV2("addr22.example.com", 8022, HEALTHY, 22); - LocalityLbEndpoints localityLbEndpoints2 = buildLocalityLbEndpointsV2( - "region2", "zone2", "subzone2", - ImmutableList.of(endpoint21, endpoint22), - 2, - 0); - - LbEndpoint endpoint31 = buildLbEndpointV2("addr31.example.com", 8031, HEALTHY, 31); - LocalityLbEndpoints localityLbEndpoints3 = buildLocalityLbEndpointsV2( - "region3", "zone3", "subzone3", - ImmutableList.of(endpoint31), - 3, - 0); - - ClusterLoadAssignment clusterLoadAssignment = buildClusterLoadAssignmentV2( - CLUSTER_NAME, - ImmutableList.of(localityLbEndpoints1, localityLbEndpoints2, localityLbEndpoints3), - ImmutableList.of()); - deliverClusterLoadAssignments(clusterLoadAssignment); - - assertThat(childBalancers).hasSize(3); - verify(childBalancers.get("subzone1")).handleResolvedAddresses( - argThat(RoundRobinBackendsMatcher.builder() - .addHostAndPort("addr11.example.com", 8011) - .addHostAndPort("addr12.example.com", 8012) - .build())); - verify(childBalancers.get("subzone2")).handleResolvedAddresses( - argThat(RoundRobinBackendsMatcher.builder() - .addHostAndPort("addr21.example.com", 8021) - .addHostAndPort("addr22.example.com", 8022) - .build())); - verify(childBalancers.get("subzone3")).handleResolvedAddresses( - argThat(RoundRobinBackendsMatcher.builder() - .addHostAndPort("addr31.example.com", 8031) - .build())); - assertThat(childHelpers).hasSize(3); - Helper childHelper2 = childHelpers.get("subzone2"); - final Subchannel subchannel = mock(Subchannel.class); - SubchannelPicker picker = new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withSubchannel(subchannel); - } - }; - verify(helper, never()).updateBalancingState(eq(READY), any(SubchannelPicker.class)); - childHelper2.updateBalancingState(READY, picker); - assertLatestSubchannelPicker(subchannel); - } - - // Uses a fake LocalityStoreFactory that creates a mock LocalityStore, and verifies interaction - // between the EDS balancer and LocalityStore. - @Test - public void edsResourceUpdate_endpointAssignmentChange() { - final ArrayDeque localityStores = new ArrayDeque<>(); - localityStoreFactory = new LocalityStoreFactory() { - @Override - LocalityStore newLocalityStore( - InternalLogId logId, - Helper helper, - LoadBalancerRegistry lbRegistry, - LoadStatsStore loadStatsStore) { - // Note that this test approach can not verify anything about how localityStore will use the - // helper in the arguments to delegate updates from localityStore to the EDS balancer, and - // can not verify anything about how loadStatsStore updates localities and drop information. - // To cover the gap, some non-exhaustive tests like - // handleAllDropUpdates_pickersAreDropped() and - // handleLocalityAssignmentUpdates_pickersUpdatedFromChildBalancer()are added to verify some - // very basic behaviors. - LocalityStore localityStore = mock(LocalityStore.class); - localityStores.add(localityStore); - return localityStore; - } - }; - edsLb = - new EdsLoadBalancer(helper, lbRegistry, localityStoreFactory, bootstrapper, channelFactory); - - deliverResolvedAddresses("edsServiceName1", null, fakeLocalityPickingPolicy, - fakeEndpointPickingPolicy); - assertThat(localityStores).hasSize(1); - LocalityStore localityStore = localityStores.peekLast(); - - ClusterLoadAssignment clusterLoadAssignment = buildClusterLoadAssignmentV2( - "edsServiceName1", - ImmutableList.of( - buildLocalityLbEndpointsV2("region1", "zone1", "subzone1", - ImmutableList.of( - buildLbEndpointV2("192.168.0.1", 8080, HEALTHY, 2)), - 1, 0)), - ImmutableList.of( - buildDropOverloadV2("cat_1", 3), - buildDropOverloadV2("cat_2", 456))); - deliverClusterLoadAssignments(clusterLoadAssignment); - EndpointUpdate endpointUpdate = getEndpointUpdateFromClusterAssignmentV2(clusterLoadAssignment); - verify(localityStore).updateDropPercentage(endpointUpdate.getDropPolicies()); - verify(localityStore).updateLocalityStore(endpointUpdate.getLocalityLbEndpointsMap()); - - clusterLoadAssignment = buildClusterLoadAssignmentV2( - "edsServiceName1", - ImmutableList.of( - buildLocalityLbEndpointsV2("region1", "zone1", "subzone1", - ImmutableList.of( - buildLbEndpointV2("192.168.0.1", 8080, HEALTHY, 2), - buildLbEndpointV2("192.168.0.1", 8088, HEALTHY, 2)), - 1, 0)), - ImmutableList.of( - buildDropOverloadV2("cat_1", 3), - buildDropOverloadV2("cat_3", 4))); - deliverClusterLoadAssignments(clusterLoadAssignment); - - endpointUpdate = getEndpointUpdateFromClusterAssignmentV2(clusterLoadAssignment); - verify(localityStore).updateDropPercentage(endpointUpdate.getDropPolicies()); - verify(localityStore).updateLocalityStore(endpointUpdate.getLocalityLbEndpointsMap()); - - // Change cluster name. - deliverResolvedAddresses("edsServiceName2", null, fakeLocalityPickingPolicy, - fakeEndpointPickingPolicy); - assertThat(localityStores).hasSize(2); - localityStore = localityStores.peekLast(); - - clusterLoadAssignment = buildClusterLoadAssignmentV2( - "edsServiceName2", - ImmutableList.of( - buildLocalityLbEndpointsV2("region2", "zone2", "subzone2", - ImmutableList.of( - buildLbEndpointV2("192.168.0.2", 8080, HEALTHY, 2), - buildLbEndpointV2("192.168.0.2", 8088, HEALTHY, 2)), - 1, 0)), - ImmutableList.of( - buildDropOverloadV2("cat_1", 3), - buildDropOverloadV2("cat_3", 4))); - deliverClusterLoadAssignments(clusterLoadAssignment); - endpointUpdate = getEndpointUpdateFromClusterAssignmentV2(clusterLoadAssignment); - verify(localityStore).updateDropPercentage(endpointUpdate.getDropPolicies()); - verify(localityStore).updateLocalityStore(endpointUpdate.getLocalityLbEndpointsMap()); - } - - @Test - public void edsResourceNotExist() { - deliverResolvedAddresses(null, null, fakeLocalityPickingPolicy, fakeEndpointPickingPolicy); - - // Forwarding 20 seconds so that the xds client will deem EDS resource not available. - fakeClock.forwardTime(20, TimeUnit.SECONDS); - assertThat(childBalancers).isEmpty(); - verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); - PickResult result = pickerCaptor.getValue().pickSubchannel(mock(PickSubchannelArgs.class)); - assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAVAILABLE); - assertThat(result.getStatus().getDescription()) - .isEqualTo("Resource " + CLUSTER_NAME + " is unavailable"); - } - - @Test - public void edsResourceRemoved() { - deliverResolvedAddresses(null, null, fakeLocalityPickingPolicy, fakeEndpointPickingPolicy); - ClusterLoadAssignment clusterLoadAssignment = - buildClusterLoadAssignmentV2(CLUSTER_NAME, - ImmutableList.of( - buildLocalityLbEndpointsV2("region", "zone", "subzone", - ImmutableList.of( - buildLbEndpointV2("192.168.0.1", 8080, HEALTHY, 2)), - 1, 0)), - ImmutableList.of()); - deliverClusterLoadAssignments(clusterLoadAssignment); - - assertThat(childBalancers).hasSize(1); - assertThat(childHelpers).hasSize(1); - LoadBalancer localityBalancer = childBalancers.get("subzone"); - Helper localityBalancerHelper = childHelpers.get("subzone"); - final Subchannel subchannel = mock(Subchannel.class); - SubchannelPicker picker = new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withSubchannel(subchannel); - } - }; - localityBalancerHelper.updateBalancingState(READY, picker); - verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture()); - PickResult result = pickerCaptor.getValue().pickSubchannel(mock(PickSubchannelArgs.class)); - assertThat(result.getSubchannel()).isSameInstanceAs(subchannel); - - // The whole cluster is no longer accessible. - // Note that EDS resource removal is achieved by CDS resource update. - responseObserver.onNext( - buildDiscoveryResponseV2( - String.valueOf(versionIno++), - Collections.emptyList(), - XdsClientImpl.ADS_TYPE_URL_CDS_V2, - String.valueOf(nonce++))); - - verify(localityBalancer).shutdown(); - verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); - result = pickerCaptor.getValue().pickSubchannel(mock(PickSubchannelArgs.class)); - assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAVAILABLE); - assertThat(result.getStatus().getDescription()) - .isEqualTo("Resource " + CLUSTER_NAME + " is unavailable"); - } - - @Test - public void transientError_noPreviousEndpointUpdateReceived() { - deliverResolvedAddresses(null, null, fakeLocalityPickingPolicy, fakeEndpointPickingPolicy); - - // Forwarding 20 seconds so that the xds client will deem EDS resource not available. - fakeClock.forwardTime(20, TimeUnit.SECONDS); - verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); - } - - @Test - public void transientError_withPreviousEndpointUpdateReceived() { - deliverResolvedAddresses(null, null, fakeLocalityPickingPolicy, fakeEndpointPickingPolicy); - // Endpoint update received. - ClusterLoadAssignment clusterLoadAssignment = - buildClusterLoadAssignmentV2(CLUSTER_NAME, - ImmutableList.of( - buildLocalityLbEndpointsV2("region1", "zone1", "subzone1", - ImmutableList.of( - buildLbEndpointV2("192.168.0.1", 8080, HEALTHY, 2)), - 1, 0)), - ImmutableList.of(buildDropOverloadV2("throttle", 1000))); - deliverClusterLoadAssignments(clusterLoadAssignment); - - verify(helper, never()).updateBalancingState( - eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); - - // XdsClient stream receives an error. - responseObserver.onError(new RuntimeException("fake error")); - verify(helper, never()).updateBalancingState( - eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); - } - - /** - * Converts ClusterLoadAssignment data to {@link EndpointUpdate}. All the needed data, that is - * clusterName, localityLbEndpointsMap and dropPolicies, is extracted from ClusterLoadAssignment, - * and all other data is ignored. - */ - private static EndpointUpdate getEndpointUpdateFromClusterAssignmentV2( - ClusterLoadAssignment clusterLoadAssignment) { - EndpointUpdate.Builder endpointUpdateBuilder = EndpointUpdate.newBuilder(); - endpointUpdateBuilder.setClusterName(clusterLoadAssignment.getClusterName()); - for (DropOverload dropOverload : clusterLoadAssignment.getPolicy().getDropOverloadsList()) { - endpointUpdateBuilder.addDropPolicy( - EnvoyProtoData.DropOverload.fromEnvoyProtoDropOverloadV2(dropOverload)); - } - for (LocalityLbEndpoints localityLbEndpoints : clusterLoadAssignment.getEndpointsList()) { - endpointUpdateBuilder.addLocalityLbEndpoints( - EnvoyProtoData.Locality.fromEnvoyProtoLocalityV2( - localityLbEndpoints.getLocality()), - EnvoyProtoData.LocalityLbEndpoints.fromEnvoyProtoLocalityLbEndpointsV2( - localityLbEndpoints)); - } - return endpointUpdateBuilder.build(); - } - - private void deliverResolvedAddresses( - @Nullable String edsServiceName, - @Nullable String lrsServerName, - PolicySelection locaityPickingPolicy, - PolicySelection endpointPickingPolicy) { - EdsConfig config = - new EdsConfig(CLUSTER_NAME, edsServiceName, lrsServerName, locaityPickingPolicy, - endpointPickingPolicy); - ResolvedAddresses.Builder resolvedAddressBuilder = ResolvedAddresses.newBuilder() - .setAddresses(ImmutableList.of()) - .setLoadBalancingPolicyConfig(config); - if (isFullFlow) { - resolvedAddressBuilder.setAttributes( - Attributes.newBuilder().set(XdsAttributes.XDS_CLIENT_POOL, - xdsClientPoolFromResolveAddresses).build()); - } - edsLb.handleResolvedAddresses(resolvedAddressBuilder.build()); - } - - private void deliverClusterLoadAssignments(ClusterLoadAssignment clusterLoadAssignment) { - responseObserver.onNext( - buildDiscoveryResponseV2( - String.valueOf(versionIno++), - ImmutableList.of(Any.pack(clusterLoadAssignment)), - XdsClientImpl.ADS_TYPE_URL_EDS_V2, - String.valueOf(nonce++))); - } - - private void assertLatestConnectivityState(ConnectivityState expectedState) { - verify(helper, atLeastOnce()).updateBalancingState( - connectivityStateCaptor.capture(), pickerCaptor.capture()); - assertThat(connectivityStateCaptor.getValue()).isEqualTo(expectedState); - } - - private void assertLatestSubchannelPicker(Subchannel expectedSubchannelToPick) { - assertLatestConnectivityState(READY); - assertThat( - pickerCaptor.getValue().pickSubchannel(mock(PickSubchannelArgs.class)).getSubchannel()) - .isEqualTo(expectedSubchannelToPick); - } - - /** - * Matcher of ResolvedAddresses for round robin load balancer based on the set of backends. - */ - private static final class RoundRobinBackendsMatcher - implements ArgumentMatcher { - - final List socketAddresses; - - RoundRobinBackendsMatcher(List socketAddresses) { - this.socketAddresses = socketAddresses; - } - - @Override - public boolean matches(ResolvedAddresses argument) { - List backends = new ArrayList<>(); - for (EquivalentAddressGroup eag : argument.getAddresses()) { - backends.add(Iterables.getOnlyElement(eag.getAddresses())); - } - return socketAddresses.equals(backends); - } - - static Builder builder() { - return new Builder(); - } - - static final class Builder { - final List socketAddresses = new ArrayList<>(); - - Builder addHostAndPort(String host, int port) { - socketAddresses.add(new InetSocketAddress(host, port)); - return this; - } - - RoundRobinBackendsMatcher build() { - return new RoundRobinBackendsMatcher(socketAddresses); - } - } - } - - /** - * A fake ObjectPool of XdsClient that keeps track of invocation times of getObject() and - * returnObject(). - */ - private static final class FakeXdsClientPool implements ObjectPool { - final XdsClient xdsClient; - int timesGetObjectCalled; - int timesReturnObjectCalled; - - FakeXdsClientPool(XdsClient xdsClient) { - this.xdsClient = xdsClient; - } - - @Override - public synchronized XdsClient getObject() { - timesGetObjectCalled++; - return xdsClient; - } - - @Override - public synchronized XdsClient returnObject(Object object) { - timesReturnObjectCalled++; - assertThat(timesReturnObjectCalled).isAtMost(timesGetObjectCalled); - return null; - } - } -} diff --git a/xds/src/test/java/io/grpc/xds/LocalityStoreTest.java b/xds/src/test/java/io/grpc/xds/LocalityStoreTest.java deleted file mode 100644 index f514cf1f1d6..00000000000 --- a/xds/src/test/java/io/grpc/xds/LocalityStoreTest.java +++ /dev/null @@ -1,1378 +0,0 @@ -/* - * Copyright 2019 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.xds; - -import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.truth.Truth.assertThat; -import static io.grpc.ConnectivityState.CONNECTING; -import static io.grpc.ConnectivityState.READY; -import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; -import static io.grpc.Status.UNAVAILABLE; -import static io.grpc.xds.XdsSubchannelPickers.BUFFER_PICKER; -import static org.mockito.AdditionalAnswers.delegatesTo; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.argThat; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.isA; -import static org.mockito.ArgumentMatchers.same; -import static org.mockito.Mockito.atLeastOnce; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.inOrder; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; -import io.grpc.ChannelLogger; -import io.grpc.ClientStreamTracer; -import io.grpc.ConnectivityState; -import io.grpc.EquivalentAddressGroup; -import io.grpc.InternalLogId; -import io.grpc.LoadBalancer; -import io.grpc.LoadBalancer.Helper; -import io.grpc.LoadBalancer.PickResult; -import io.grpc.LoadBalancer.PickSubchannelArgs; -import io.grpc.LoadBalancer.ResolvedAddresses; -import io.grpc.LoadBalancer.Subchannel; -import io.grpc.LoadBalancer.SubchannelPicker; -import io.grpc.LoadBalancerProvider; -import io.grpc.LoadBalancerRegistry; -import io.grpc.Status; -import io.grpc.Status.Code; -import io.grpc.SynchronizationContext; -import io.grpc.internal.FakeClock; -import io.grpc.internal.FakeClock.ScheduledTask; -import io.grpc.internal.FakeClock.TaskFilter; -import io.grpc.xds.ClientLoadCounter.LoadRecordingStreamTracerFactory; -import io.grpc.xds.ClientLoadCounter.MetricsRecordingListener; -import io.grpc.xds.EnvoyProtoData.ClusterStats; -import io.grpc.xds.EnvoyProtoData.DropOverload; -import io.grpc.xds.EnvoyProtoData.LbEndpoint; -import io.grpc.xds.EnvoyProtoData.Locality; -import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints; -import io.grpc.xds.LoadStatsManager.LoadStatsStore; -import io.grpc.xds.LocalityStore.LocalityStoreImpl; -import io.grpc.xds.OrcaOobUtil.OrcaOobReportListener; -import io.grpc.xds.OrcaOobUtil.OrcaReportingConfig; -import io.grpc.xds.OrcaOobUtil.OrcaReportingHelperWrapper; -import io.grpc.xds.OrcaPerRequestUtil.OrcaPerRequestReportListener; -import io.grpc.xds.WeightedRandomPicker.WeightedChildPicker; -import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; -import java.net.InetSocketAddress; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.mockito.ArgumentCaptor; -import org.mockito.ArgumentMatcher; -import org.mockito.InOrder; -import org.mockito.Mock; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; -import org.mockito.stubbing.Answer; - -/** - * Tests for {@link LocalityStore}. - */ -@RunWith(JUnit4.class) -public class LocalityStoreTest { - @Rule - public final MockitoRule mockitoRule = MockitoJUnit.rule(); - - private final SynchronizationContext syncContext = new SynchronizationContext( - new Thread.UncaughtExceptionHandler() { - @Override - public void uncaughtException(Thread t, Throwable e) { - throw new AssertionError(e); - } - }); - - private final InternalLogId logId = InternalLogId.allocate("locality-store-test", null); - private final LoadBalancerRegistry lbRegistry = new LoadBalancerRegistry(); - private final Map loadBalancers = new HashMap<>(); - private final Map childHelpers = new HashMap<>(); - private final Map childHelperWrappers = new HashMap<>(); - private final FakeClock fakeClock = new FakeClock(); - - private final TaskFilter deactivationTaskFilter = new TaskFilter() { - @Override - public boolean shouldAccept(Runnable runnable) { - return runnable.toString().contains("DeletionTask"); - } - }; - - private final TaskFilter failOverTaskFilter = new TaskFilter() { - @Override - public boolean shouldAccept(Runnable runnable) { - return runnable.toString().contains("FailOverTask"); - } - }; - - private final LoadBalancerProvider lbProvider = new LoadBalancerProvider() { - - @Override - public boolean isAvailable() { - return true; - } - - @Override - public int getPriority() { - return 0; - } - - @Override - public String getPolicyName() { - return "round_robin"; - } - - @Override - public LoadBalancer newLoadBalancer(Helper helper) { - LoadBalancer fakeLb = mock(LoadBalancer.class); - loadBalancers.put(helper.getAuthority(), fakeLb); - childHelpers.put(helper.getAuthority(), helper); - return fakeLb; - } - }; - - private final Locality locality1 = new Locality("r1", "z1", "sz1"); - private final Locality locality2 = new Locality("r2", "z2", "sz2"); - private final Locality locality3 = new Locality("r3", "z3", "sz3"); - private final Locality locality4 = new Locality("r4", "z4", "sz4"); - - private final EquivalentAddressGroup eag11 = - new EquivalentAddressGroup(new InetSocketAddress("addr11", 11)); - private final EquivalentAddressGroup eag12 = - new EquivalentAddressGroup(new InetSocketAddress("addr12", 12)); - private final EquivalentAddressGroup eag21 = - new EquivalentAddressGroup(new InetSocketAddress("addr21", 21)); - private final EquivalentAddressGroup eag22 = - new EquivalentAddressGroup(new InetSocketAddress("addr22", 22)); - private final EquivalentAddressGroup eag31 = - new EquivalentAddressGroup(new InetSocketAddress("addr31", 31)); - private final EquivalentAddressGroup eag32 = - new EquivalentAddressGroup(new InetSocketAddress("addr32", 32)); - private final EquivalentAddressGroup eag41 = - new EquivalentAddressGroup(new InetSocketAddress("addr41", 41)); - private final EquivalentAddressGroup eag42 = - new EquivalentAddressGroup(new InetSocketAddress("addr42", 42)); - - private final LbEndpoint lbEndpoint11 = new LbEndpoint(eag11, 11, true); - private final LbEndpoint lbEndpoint12 = new LbEndpoint(eag12, 12, true); - private final LbEndpoint lbEndpoint21 = new LbEndpoint(eag21, 21, true); - private final LbEndpoint lbEndpoint22 = new LbEndpoint(eag22, 22, true); - private final LbEndpoint lbEndpoint31 = new LbEndpoint(eag31, 31, true); - private final LbEndpoint lbEndpoint32 = new LbEndpoint(eag32, 32, true); - private final LbEndpoint lbEndpoint41 = new LbEndpoint(eag41, 41, true); - private final LbEndpoint lbEndpoint42 = new LbEndpoint(eag42, 42, true); - - private final Map namedLocalities - = ImmutableMap.of("sz1", locality1, "sz2", locality2, "sz3", locality3, "sz4", locality4); - - @Mock - private Helper helper; - @Mock - private PickSubchannelArgs pickSubchannelArgs; - @Mock - private ThreadSafeRandom random; - @Mock - private OrcaPerRequestUtil orcaPerRequestUtil; - @Mock - private OrcaOobUtil orcaOobUtil; - private final FakeLoadStatsStore fakeLoadStatsStore = new FakeLoadStatsStore(); - private final LoadStatsStore loadStatsStore = - mock(LoadStatsStore.class, delegatesTo(fakeLoadStatsStore)); - - private LocalityStore localityStore; - - @Before - public void setUp() { - doReturn(mock(ChannelLogger.class)).when(helper).getChannelLogger(); - doReturn(syncContext).when(helper).getSynchronizationContext(); - doReturn(fakeClock.getScheduledExecutorService()).when(helper).getScheduledExecutorService(); - when(orcaOobUtil.newOrcaReportingHelperWrapper(any(Helper.class), - any(OrcaOobReportListener.class))) - .thenAnswer(new Answer() { - @Override - public OrcaReportingHelperWrapper answer(InvocationOnMock invocation) { - Helper h = invocation.getArgument(0); - FakeOrcaReportingHelperWrapper res = - new FakeOrcaReportingHelperWrapper(h); - childHelperWrappers.put(h.getAuthority(), res); - return res; - } - }); - lbRegistry.register(lbProvider); - localityStore = - new LocalityStoreImpl(logId, helper, lbRegistry, random, loadStatsStore, - orcaPerRequestUtil, orcaOobUtil); - } - - @Test - public void updateLocalityStore_pickResultInterceptedForLoadRecordingWhenSubchannelReady() { - // Simulate receiving two localities. - LocalityLbEndpoints localityInfo1 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint11, lbEndpoint12), 1, 0); - LocalityLbEndpoints localityInfo2 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint21, lbEndpoint22), 2, 0); - localityStore.updateLocalityStore(ImmutableMap.of( - locality1, localityInfo1, locality2, localityInfo2)); - - // Two child balancers are created. - assertThat(loadBalancers).hasSize(2); - - ClientStreamTracer.Factory metricsTracingFactory1 = mock(ClientStreamTracer.Factory.class); - ClientStreamTracer.Factory metricsTracingFactory2 = mock(ClientStreamTracer.Factory.class); - when(orcaPerRequestUtil.newOrcaClientStreamTracerFactory(any(ClientStreamTracer.Factory.class), - any(OrcaPerRequestReportListener.class))) - .thenReturn(metricsTracingFactory1, metricsTracingFactory2); - - Subchannel subchannel1 = mock(Subchannel.class); - Subchannel subchannel2 = mock(Subchannel.class); - final PickResult result1 = PickResult.withSubchannel(subchannel1); - final PickResult result2 = - PickResult.withSubchannel(subchannel2, mock(ClientStreamTracer.Factory.class)); - SubchannelPicker subchannelPicker1 = mock(SubchannelPicker.class); - SubchannelPicker subchannelPicker2 = mock(SubchannelPicker.class); - when(subchannelPicker1.pickSubchannel(any(PickSubchannelArgs.class))) - .thenReturn(result1); - when(subchannelPicker2.pickSubchannel(any(PickSubchannelArgs.class))) - .thenReturn(result2); - - // Simulate picker updates for the two localities with dummy pickers. - childHelpers.get("sz1").updateBalancingState(READY, subchannelPicker1); - childHelpers.get("sz2").updateBalancingState(READY, subchannelPicker2); - - ArgumentCaptor interLocalityPickerCaptor = ArgumentCaptor.forClass(null); - verify(helper, times(2)).updateBalancingState(eq(READY), interLocalityPickerCaptor.capture()); - WeightedRandomPicker interLocalityPicker = - (WeightedRandomPicker) interLocalityPickerCaptor.getValue(); - assertThat(interLocalityPicker.weightedChildPickers).hasSize(2); - - // Verify each PickResult picked is intercepted with client stream tracer factory for - // recording load and backend metrics. - Map localitiesBySubchannel - = ImmutableMap.of(subchannel1, locality1, subchannel2, locality2); - Map metricsTracingFactoriesBySubchannel - = ImmutableMap.of(subchannel1, metricsTracingFactory1, subchannel2, metricsTracingFactory2); - for (int i = 0; i < interLocalityPicker.weightedChildPickers.size(); i++) { - PickResult pickResult = interLocalityPicker.weightedChildPickers.get(i).getPicker() - .pickSubchannel(pickSubchannelArgs); - Subchannel expectedSubchannel = pickResult.getSubchannel(); - Locality expectedLocality = localitiesBySubchannel.get(expectedSubchannel); - ArgumentCaptor listenerCaptor = ArgumentCaptor.forClass(null); - verify(orcaPerRequestUtil, times(i + 1)) - .newOrcaClientStreamTracerFactory(any(ClientStreamTracer.Factory.class), - listenerCaptor.capture()); - assertThat(listenerCaptor.getValue()).isInstanceOf(MetricsRecordingListener.class); - MetricsRecordingListener listener = (MetricsRecordingListener) listenerCaptor.getValue(); - assertThat(listener.getCounter()) - .isSameInstanceAs(fakeLoadStatsStore.localityCounters.get(expectedLocality)); - assertThat(pickResult.getStreamTracerFactory()) - .isInstanceOf(LoadRecordingStreamTracerFactory.class); - LoadRecordingStreamTracerFactory loadRecordingFactory = - (LoadRecordingStreamTracerFactory) pickResult.getStreamTracerFactory(); - assertThat(loadRecordingFactory.getCounter()) - .isSameInstanceAs(fakeLoadStatsStore.localityCounters.get(expectedLocality)); - assertThat(loadRecordingFactory.delegate()) - .isSameInstanceAs(metricsTracingFactoriesBySubchannel.get(expectedSubchannel)); - } - } - - @Test - public void childLbPerformOobBackendMetricsAggregation() { - // Simulate receiving two localities. - LocalityLbEndpoints localityInfo1 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint11, lbEndpoint12), 1, 0); - LocalityLbEndpoints localityInfo2 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint21, lbEndpoint22), 2, 0); - localityStore.updateLocalityStore(ImmutableMap.of( - locality1, localityInfo1, locality2, localityInfo2)); - - // Two child balancers are created. - assertThat(loadBalancers).hasSize(2); - assertThat(childHelperWrappers).hasSize(2); - - class HelperMatcher implements ArgumentMatcher { - - private final String authority; - - private HelperMatcher(String authority) { - this.authority = checkNotNull(authority, "authority"); - } - - @Override - public boolean matches(Helper argument) { - return authority.equals(argument.getAuthority()); - } - } - - Map localities = ImmutableMap.of("sz1", locality1, "sz2", locality2); - for (Helper h : childHelpers.values()) { - ArgumentCaptor listenerCaptor = ArgumentCaptor.forClass(null); - verify(orcaOobUtil) - .newOrcaReportingHelperWrapper(argThat(new HelperMatcher(h.getAuthority())), - listenerCaptor.capture()); - assertThat(listenerCaptor.getValue()).isInstanceOf(MetricsRecordingListener.class); - MetricsRecordingListener listener = (MetricsRecordingListener) listenerCaptor.getValue(); - assertThat(listener.getCounter()) - .isSameInstanceAs(fakeLoadStatsStore - .localityCounters.get(localities.get(h.getAuthority()))); - } - - // Simulate receiving updates for backend metrics reporting interval. - localityStore.updateOobMetricsReportInterval(1952); - for (FakeOrcaReportingHelperWrapper orcaWrapper : childHelperWrappers.values()) { - assertThat(orcaWrapper.reportIntervalNanos).isEqualTo(1952); - } - - localityStore.updateOobMetricsReportInterval(9251); - for (FakeOrcaReportingHelperWrapper orcaWrapper : childHelperWrappers.values()) { - assertThat(orcaWrapper.reportIntervalNanos).isEqualTo(9251); - } - } - - @Test - public void updateOobMetricsReportIntervalBeforeChildLbCreated() { - // Simulate receiving update for backend metrics reporting interval. - localityStore.updateOobMetricsReportInterval(1952); - - assertThat(loadBalancers).isEmpty(); - - // Simulate receiving two localities. - LocalityLbEndpoints localityInfo1 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint11, lbEndpoint12), 1, 0); - LocalityLbEndpoints localityInfo2 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint21, lbEndpoint22), 2, 0); - localityStore.updateLocalityStore(ImmutableMap.of( - locality1, localityInfo1, locality2, localityInfo2)); - - // Two child balancers are created. - assertThat(loadBalancers).hasSize(2); - assertThat(childHelperWrappers).hasSize(2); - - for (FakeOrcaReportingHelperWrapper orcaWrapper : childHelperWrappers.values()) { - assertThat(orcaWrapper.reportIntervalNanos).isEqualTo(1952); - } - } - - @Test - public void updateLoaclityStore_withEmptyDropList() { - localityStore.updateDropPercentage(ImmutableList.of()); - LocalityLbEndpoints localityInfo1 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint11, lbEndpoint12), 1, 0); - LocalityLbEndpoints localityInfo2 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint21, lbEndpoint22), 2, 0); - LocalityLbEndpoints localityInfo3 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint31, lbEndpoint32), 3, 0); - ImmutableMap localityInfoMap = ImmutableMap.of( - locality1, localityInfo1, locality2, localityInfo2, locality3, localityInfo3); - localityStore.updateLocalityStore(localityInfoMap); - verify(helper).updateBalancingState(CONNECTING, BUFFER_PICKER); - - assertThat(loadBalancers).hasSize(3); - assertThat(loadBalancers.keySet()).containsExactly("sz1", "sz2", "sz3"); - ArgumentCaptor resolvedAddressesCaptor1 = - ArgumentCaptor.forClass(ResolvedAddresses.class); - verify(loadBalancers.get("sz1")).handleResolvedAddresses(resolvedAddressesCaptor1.capture()); - assertThat(resolvedAddressesCaptor1.getValue().getAddresses()).containsExactly(eag11, eag12); - ArgumentCaptor resolvedAddressesCaptor2 = - ArgumentCaptor.forClass(ResolvedAddresses.class); - verify(loadBalancers.get("sz2")).handleResolvedAddresses(resolvedAddressesCaptor2.capture()); - assertThat(resolvedAddressesCaptor2.getValue().getAddresses()).containsExactly(eag21, eag22); - ArgumentCaptor resolvedAddressesCaptor3 = - ArgumentCaptor.forClass(ResolvedAddresses.class); - verify(loadBalancers.get("sz3")).handleResolvedAddresses(resolvedAddressesCaptor3.capture()); - assertThat(resolvedAddressesCaptor3.getValue().getAddresses()).containsExactly(eag31, eag32); - // verify no more updateBalancingState except the initial CONNECTING state - verify(helper, times(1)).updateBalancingState( - any(ConnectivityState.class), any(SubchannelPicker.class)); - - // subchannel12 goes to CONNECTING - final Subchannel subchannel12 = mock(Subchannel.class); - SubchannelPicker subchannelPicker12 = new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withSubchannel(subchannel12); - } - }; - childHelpers.get("sz1").updateBalancingState(CONNECTING, subchannelPicker12); - ArgumentCaptor subchannelPickerCaptor12 = - ArgumentCaptor.forClass(SubchannelPicker.class); - verify(helper, times(2)).updateBalancingState( - same(CONNECTING), subchannelPickerCaptor12.capture()); - assertThat(subchannelPickerCaptor12.getValue().pickSubchannel(pickSubchannelArgs)) - .isEqualTo(PickResult.withNoResult()); - - // subchannel31 goes to READY - final Subchannel subchannel31 = mock(Subchannel.class); - SubchannelPicker subchannelPicker31 = new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withSubchannel(subchannel31); - } - }; - childHelpers.get("sz3").updateBalancingState(READY, subchannelPicker31); - ArgumentCaptor subchannelPickerCaptor = - ArgumentCaptor.forClass(null); - verify(helper).updateBalancingState(same(READY), subchannelPickerCaptor.capture()); - WeightedRandomPicker interLocalityPicker = - (WeightedRandomPicker) subchannelPickerCaptor.getValue(); - assertThat(interLocalityPicker.weightedChildPickers).hasSize(1); - assertThat(interLocalityPicker.pickSubchannel(pickSubchannelArgs).getSubchannel()) - .isEqualTo(subchannel31); - - // subchannel12 goes to READY - childHelpers.get("sz1").updateBalancingState(READY, subchannelPicker12); - verify(helper, times(2)).updateBalancingState(same(READY), subchannelPickerCaptor.capture()); - interLocalityPicker = (WeightedRandomPicker) subchannelPickerCaptor.getValue(); - assertThat(interLocalityPicker.weightedChildPickers).hasSize(2); - - Set pickedReadySubchannels = new HashSet<>(); - for (WeightedChildPicker weightedPicker : interLocalityPicker.weightedChildPickers) { - PickResult result = weightedPicker.getPicker().pickSubchannel(pickSubchannelArgs); - pickedReadySubchannels.add(result.getSubchannel()); - } - assertThat(pickedReadySubchannels).containsExactly(subchannel31, subchannel12); - - // update with new addresses - localityInfo1 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint11), 1, 0); - LocalityLbEndpoints localityInfo4 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint41, lbEndpoint42), 4, 0); - localityInfoMap = ImmutableMap.of( - locality2, localityInfo2, locality4, localityInfo4, locality1, localityInfo1); - localityStore.updateLocalityStore(localityInfoMap); - - assertThat(loadBalancers).hasSize(4); - verify(loadBalancers.get("sz2"), times(2)) - .handleResolvedAddresses(resolvedAddressesCaptor2.capture()); - assertThat(resolvedAddressesCaptor2.getValue().getAddresses()).containsExactly(eag21, eag22); - ArgumentCaptor resolvedAddressesCaptor4 = - ArgumentCaptor.forClass(ResolvedAddresses.class); - verify(loadBalancers.get("sz4")).handleResolvedAddresses(resolvedAddressesCaptor4.capture()); - assertThat(resolvedAddressesCaptor4.getValue().getAddresses()).containsExactly(eag41, eag42); - verify(loadBalancers.get("sz1"), times(2)) - .handleResolvedAddresses(resolvedAddressesCaptor1.capture()); - assertThat(resolvedAddressesCaptor1.getValue().getAddresses()).containsExactly(eag11); - verify(helper, times(3)).updateBalancingState(same(READY), subchannelPickerCaptor.capture()); - interLocalityPicker = (WeightedRandomPicker) subchannelPickerCaptor.getValue(); - assertThat(interLocalityPicker.weightedChildPickers).hasSize(1); - - fakeClock.forwardTime(14, TimeUnit.MINUTES); - verify(loadBalancers.get("sz3"), never()).shutdown(); - fakeClock.forwardTime(1, TimeUnit.MINUTES); - verify(loadBalancers.get("sz3")).shutdown(); - - verify(random, never()).nextInt(1000_000); - } - - @Test - public void updateLoaclityStore_deactivateAndReactivate() { - LocalityLbEndpoints localityInfo1 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint11, lbEndpoint12), 1, 0); - LocalityLbEndpoints localityInfo2 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint21, lbEndpoint22), 2, 0); - LocalityLbEndpoints localityInfo3 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint31, lbEndpoint32), 3, 0); - ImmutableMap localityInfoMap = ImmutableMap.of( - locality1, localityInfo1, locality2, localityInfo2, locality3, localityInfo3); - localityStore.updateLocalityStore(localityInfoMap); - - assertThat(loadBalancers.keySet()).containsExactly("sz1", "sz2", "sz3"); - - LoadBalancer lb1 = loadBalancers.get("sz1"); - LoadBalancer lb2 = loadBalancers.get("sz2"); - LoadBalancer lb3 = loadBalancers.get("sz3"); - - final Subchannel subchannel1 = mock(Subchannel.class); - SubchannelPicker subchannelPicker1 = new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withSubchannel(subchannel1); - } - }; - childHelpers.get("sz1").updateBalancingState(READY, subchannelPicker1); - final Subchannel subchannel3 = mock(Subchannel.class); - SubchannelPicker subchannelPicker3 = new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withSubchannel(subchannel3); - } - }; - childHelpers.get("sz3").updateBalancingState(READY, subchannelPicker3); - - // update localities, removing sz1, sz2, keeping sz3, and adding sz4 - LocalityLbEndpoints localityInfo4 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint41, lbEndpoint42), 4, 0); - localityInfoMap = ImmutableMap.of(locality3, localityInfo3, locality4, localityInfo4); - localityStore.updateLocalityStore(localityInfoMap); - - assertThat(loadBalancers.keySet()).containsExactly("sz1", "sz2", "sz3", "sz4"); - - LoadBalancer lb4 = loadBalancers.get("sz4"); - - ArgumentCaptor subchannelPickerCaptor = ArgumentCaptor.forClass(null); - // helper updated multiple times. Don't care how many times, just capture the latest picker - verify(helper, atLeastOnce()).updateBalancingState( - same(READY), subchannelPickerCaptor.capture()); - WeightedRandomPicker interLocalityPicker = - (WeightedRandomPicker) subchannelPickerCaptor.getValue(); - assertThat(interLocalityPicker.weightedChildPickers).hasSize(1); - assertThat(interLocalityPicker.pickSubchannel(pickSubchannelArgs).getSubchannel()) - .isEqualTo(subchannel3); - - // verify no traffic will go to deactivated locality - final Subchannel subchannel2 = mock(Subchannel.class); - SubchannelPicker subchannelPicker2 = new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withSubchannel(subchannel2); - } - }; - childHelpers.get("sz2").updateBalancingState(READY, subchannelPicker2); - verify(helper, atLeastOnce()).updateBalancingState( - same(READY), subchannelPickerCaptor.capture()); - interLocalityPicker = - (WeightedRandomPicker) subchannelPickerCaptor.getValue(); - assertThat(interLocalityPicker.weightedChildPickers).hasSize(1); - assertThat(interLocalityPicker.pickSubchannel(pickSubchannelArgs).getSubchannel()) - .isEqualTo(subchannel3); - - // update localities, reactivating sz1 - localityInfoMap = ImmutableMap.of( - locality1, localityInfo1, locality3, localityInfo3, locality4, localityInfo4); - localityStore.updateLocalityStore(localityInfoMap); - verify(helper, atLeastOnce()).updateBalancingState( - same(READY), subchannelPickerCaptor.capture()); - interLocalityPicker = - (WeightedRandomPicker) subchannelPickerCaptor.getValue(); - assertThat(interLocalityPicker.weightedChildPickers).hasSize(2); - assertThat(interLocalityPicker.weightedChildPickers.get(0).getPicker() - .pickSubchannel(pickSubchannelArgs).getSubchannel()).isEqualTo(subchannel1); - assertThat(interLocalityPicker.weightedChildPickers.get(1).getPicker() - .pickSubchannel(pickSubchannelArgs).getSubchannel()).isEqualTo(subchannel3); - - verify(lb2, never()).shutdown(); - // delayed deletion timer expires, no reactivation - fakeClock.forwardTime(15, TimeUnit.MINUTES); - verify(lb1, never()).shutdown(); - verify(lb2).shutdown(); - // update localities, re-adding sz2, keeping sz1, and removing sz3, sz4 - localityInfoMap = ImmutableMap.of( - locality1, localityInfo1, locality2, localityInfo2); - localityStore.updateLocalityStore(localityInfoMap); - - LoadBalancer newLb2 = loadBalancers.get("sz2"); - assertThat(newLb2).isNotSameInstanceAs(lb2); - - verify(helper, atLeastOnce()).updateBalancingState( - same(READY), subchannelPickerCaptor.capture()); - interLocalityPicker = - (WeightedRandomPicker) subchannelPickerCaptor.getValue(); - assertThat(interLocalityPicker.weightedChildPickers).hasSize(1); - assertThat(interLocalityPicker.pickSubchannel(pickSubchannelArgs).getSubchannel()) - .isEqualTo(subchannel1); - // sz3, sz4 pending removal - assertThat(fakeClock.getPendingTasks(deactivationTaskFilter)).hasSize(2); - - // verify lb1, lb3, and lb4 never shutdown and never changed since created - verify(lb1, never()).shutdown(); - verify(newLb2, never()).shutdown(); - verify(lb3, never()).shutdown(); - verify(lb4, never()).shutdown(); - assertThat(loadBalancers.get("sz1")).isSameInstanceAs(lb1); - assertThat(loadBalancers.get("sz2")).isSameInstanceAs(newLb2); - assertThat(loadBalancers.get("sz3")).isSameInstanceAs(lb3); - assertThat(loadBalancers.get("sz4")).isSameInstanceAs(lb4); - - localityStore.reset(); - assertThat(fakeClock.getPendingTasks(deactivationTaskFilter)).isEmpty(); - verify(lb1).shutdown(); - verify(newLb2).shutdown(); - verify(lb3).shutdown(); - verify(lb4).shutdown(); - } - - @Test - public void updateLoaclityStore_withDrop() { - localityStore.updateDropPercentage(ImmutableList.of( - new DropOverload("throttle", 365), - new DropOverload("lb", 1234))); - LocalityLbEndpoints localityInfo1 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint11, lbEndpoint12), 1, 0); - LocalityLbEndpoints localityInfo2 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint21, lbEndpoint22), 2, 0); - LocalityLbEndpoints localityInfo3 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint31, lbEndpoint32), 3, 0); - ImmutableMap localityInfoMap = ImmutableMap.of( - locality1, localityInfo1, locality2, localityInfo2, locality3, localityInfo3); - localityStore.updateLocalityStore(localityInfoMap); - - assertThat(loadBalancers).hasSize(3); - ArgumentCaptor resolvedAddressesCaptor1 = - ArgumentCaptor.forClass(ResolvedAddresses.class); - verify(loadBalancers.get("sz1")).handleResolvedAddresses(resolvedAddressesCaptor1.capture()); - assertThat(resolvedAddressesCaptor1.getValue().getAddresses()).containsExactly(eag11, eag12); - ArgumentCaptor resolvedAddressesCaptor2 = - ArgumentCaptor.forClass(ResolvedAddresses.class); - verify(loadBalancers.get("sz2")).handleResolvedAddresses(resolvedAddressesCaptor2.capture()); - assertThat(resolvedAddressesCaptor2.getValue().getAddresses()).containsExactly(eag21, eag22); - ArgumentCaptor resolvedAddressesCaptor3 = - ArgumentCaptor.forClass(ResolvedAddresses.class); - verify(loadBalancers.get("sz3")).handleResolvedAddresses(resolvedAddressesCaptor3.capture()); - assertThat(resolvedAddressesCaptor3.getValue().getAddresses()).containsExactly(eag31, eag32); - ArgumentCaptor subchannelPickerCaptor = - ArgumentCaptor.forClass(SubchannelPicker.class); - verify(helper).updateBalancingState(same(CONNECTING), subchannelPickerCaptor.capture()); - - int times = 0; - InOrder inOrder = inOrder(loadStatsStore); - doReturn(365, 1234).when(random).nextInt(1000_000); - assertThat(subchannelPickerCaptor.getValue().pickSubchannel(pickSubchannelArgs)) - .isEqualTo(PickResult.withNoResult()); - verify(random, times(times += 2)).nextInt(1000_000); - inOrder.verify(loadStatsStore, never()).recordDroppedRequest(anyString()); - - doReturn(366, 1235).when(random).nextInt(1000_000); - assertThat(subchannelPickerCaptor.getValue().pickSubchannel(pickSubchannelArgs)) - .isEqualTo(PickResult.withNoResult()); - verify(random, times(times += 2)).nextInt(1000_000); - inOrder.verify(loadStatsStore, never()).recordDroppedRequest(anyString()); - - doReturn(364, 1234).when(random).nextInt(1000_000); - assertThat(subchannelPickerCaptor.getValue().pickSubchannel(pickSubchannelArgs).isDrop()) - .isTrue(); - verify(random, times(times += 1)).nextInt(1000_000); - inOrder.verify(loadStatsStore).recordDroppedRequest(eq("throttle")); - - doReturn(365, 1233).when(random).nextInt(1000_000); - assertThat(subchannelPickerCaptor.getValue().pickSubchannel(pickSubchannelArgs).isDrop()) - .isTrue(); - verify(random, times(times += 2)).nextInt(1000_000); - inOrder.verify(loadStatsStore).recordDroppedRequest(eq("lb")); - - // subchannel12 goes to READY - final Subchannel subchannel12 = mock(Subchannel.class); - SubchannelPicker subchannelPicker12 = new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withSubchannel(subchannel12); - } - }; - childHelpers.get("sz1").updateBalancingState(READY, subchannelPicker12); - ArgumentCaptor subchannelPickerCaptor12 = - ArgumentCaptor.forClass(SubchannelPicker.class); - verify(helper).updateBalancingState(same(READY), subchannelPickerCaptor12.capture()); - - doReturn(365, 1234).when(random).nextInt(1000_000); - assertThat(subchannelPickerCaptor12.getValue().pickSubchannel(pickSubchannelArgs) - .getSubchannel()).isEqualTo(subchannel12); - verify(random, times(times += 2)).nextInt(1000_000); - inOrder.verify(loadStatsStore, never()).recordDroppedRequest(anyString()); - - doReturn(366, 1235).when(random).nextInt(1000_000); - assertThat(subchannelPickerCaptor12.getValue().pickSubchannel(pickSubchannelArgs) - .getSubchannel()).isEqualTo(subchannel12); - verify(random, times(times += 2)).nextInt(1000_000); - inOrder.verify(loadStatsStore, never()).recordDroppedRequest(anyString()); - - doReturn(364, 1234).when(random).nextInt(1000_000); - assertThat(subchannelPickerCaptor12.getValue().pickSubchannel(pickSubchannelArgs).isDrop()) - .isTrue(); - verify(random, times(times += 1)).nextInt(1000_000); - inOrder.verify(loadStatsStore).recordDroppedRequest(eq("throttle")); - - doReturn(365, 1233).when(random).nextInt(1000_000); - assertThat(subchannelPickerCaptor12.getValue().pickSubchannel(pickSubchannelArgs).isDrop()) - .isTrue(); - verify(random, times(times + 2)).nextInt(1000_000); - inOrder.verify(loadStatsStore).recordDroppedRequest(eq("lb")); - inOrder.verifyNoMoreInteractions(); - } - - @Test - public void updateLoaclityStore_withAllDropBeforeLocalityUpdateConnectivityState() { - localityStore.updateDropPercentage(ImmutableList.of( - new DropOverload("throttle", 365), - new DropOverload("lb", 1000_000))); - LocalityLbEndpoints localityInfo1 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint11, lbEndpoint12), 1, 0); - LocalityLbEndpoints localityInfo2 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint21, lbEndpoint22), 2, 0); - LocalityLbEndpoints localityInfo3 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint31, lbEndpoint32), 3, 0); - ImmutableMap localityInfoMap = ImmutableMap.of( - locality1, localityInfo1, locality2, localityInfo2, locality3, localityInfo3); - localityStore.updateLocalityStore(localityInfoMap); - - ArgumentCaptor subchannelPickerCaptor = - ArgumentCaptor.forClass(SubchannelPicker.class); - verify(helper).updateBalancingState(same(CONNECTING), subchannelPickerCaptor.capture()); - doReturn(999_999).when(random).nextInt(1000_000); - assertThat(subchannelPickerCaptor.getValue().pickSubchannel(pickSubchannelArgs).isDrop()) - .isTrue(); - verify(random, times(2)).nextInt(1000_000); - } - - @Test - public void updateLoaclityStore_withUnHealthyEndPoints() { - LbEndpoint lbEndpoint11 = new LbEndpoint(eag11, 11, true); - LbEndpoint lbEndpoint12 = new LbEndpoint(eag12, 12, true); - LbEndpoint lbEndpoint21 = new LbEndpoint(eag21, 21, false); // unhealthy - LbEndpoint lbEndpoint22 = new LbEndpoint(eag22, 22, true); - LbEndpoint lbEndpoint31 = new LbEndpoint(eag31, 31, false); // unhealthy - LbEndpoint lbEndpoint32 = new LbEndpoint(eag32, 32, false); // unhealthy - LbEndpoint lbEndpoint41 = new LbEndpoint(eag41, 41, true); - LbEndpoint lbEndpoint42 = new LbEndpoint(eag42, 42, true); - LocalityLbEndpoints localityInfo1 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint11, lbEndpoint12), 1, 0); - LocalityLbEndpoints localityInfo2 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint21, lbEndpoint22), 2, 0); - LocalityLbEndpoints localityInfo3 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint31, lbEndpoint32), 3, 0); - LocalityLbEndpoints localityInfo4 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint41, lbEndpoint42), 4, 0); - ImmutableMap localityInfoMap = ImmutableMap.of( - locality1, localityInfo1, locality2, localityInfo2, locality3, localityInfo3, - locality4, localityInfo4); - localityStore.updateLocalityStore(localityInfoMap); - verify(helper).updateBalancingState(CONNECTING, BUFFER_PICKER); - - assertThat(loadBalancers).hasSize(4); - assertThat(loadBalancers.keySet()).containsExactly("sz1", "sz2", "sz3", "sz4"); - ArgumentCaptor resolvedAddressesCaptor1 = - ArgumentCaptor.forClass(ResolvedAddresses.class); - verify(loadBalancers.get("sz1")).handleResolvedAddresses(resolvedAddressesCaptor1.capture()); - assertThat(resolvedAddressesCaptor1.getValue().getAddresses()).containsExactly(eag11, eag12); - ArgumentCaptor resolvedAddressesCaptor2 = - ArgumentCaptor.forClass(ResolvedAddresses.class); - verify(loadBalancers.get("sz2")).handleResolvedAddresses(resolvedAddressesCaptor2.capture()); - assertThat(resolvedAddressesCaptor2.getValue().getAddresses()).containsExactly(eag22); - verify(loadBalancers.get("sz3"), never()).handleResolvedAddresses(any(ResolvedAddresses.class)); - ArgumentCaptor statusCaptor = ArgumentCaptor.forClass(Status.class); - verify(loadBalancers.get("sz3")).handleNameResolutionError(statusCaptor.capture()); - assertThat(statusCaptor.getValue().getCode()).isEqualTo(Status.Code.UNAVAILABLE); - ArgumentCaptor resolvedAddressesCaptor4 = - ArgumentCaptor.forClass(ResolvedAddresses.class); - verify(loadBalancers.get("sz4")).handleResolvedAddresses(resolvedAddressesCaptor4.capture()); - assertThat(resolvedAddressesCaptor4.getValue().getAddresses()).containsExactly(eag41, eag42); - // verify no more updateBalancingState except the initial CONNECTING state - verify(helper, times(1)).updateBalancingState( - any(ConnectivityState.class), any(SubchannelPicker.class)); - - // update with different healthy status - lbEndpoint11 = new LbEndpoint(eag11, 11, false); // unhealthy - lbEndpoint12 = new LbEndpoint(eag12, 12, false); // unhealthy - lbEndpoint21 = new LbEndpoint(eag21, 21, true); - lbEndpoint22 = new LbEndpoint(eag22, 22, false); // unhealthy - lbEndpoint31 = new LbEndpoint(eag31, 31, true); - lbEndpoint32 = new LbEndpoint(eag32, 32, false); // unhealthy - localityInfo1 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint11, lbEndpoint12), 1, 0); - localityInfo2 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint21, lbEndpoint22), 2, 0); - localityInfo3 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint31, lbEndpoint32), 3, 0); - localityInfoMap = ImmutableMap.of( - locality1, localityInfo1, locality2, localityInfo2, locality3, localityInfo3); - localityStore.updateLocalityStore(localityInfoMap); - - verify(loadBalancers.get("sz1"), times(1)) - .handleResolvedAddresses(any(ResolvedAddresses.class)); - verify(loadBalancers.get("sz1"), times(1)).handleNameResolutionError(statusCaptor.capture()); - assertThat(statusCaptor.getValue().getCode()).isEqualTo(Status.Code.UNAVAILABLE); - verify(loadBalancers.get("sz2"), times(2)) - .handleResolvedAddresses(resolvedAddressesCaptor2.capture()); - assertThat(resolvedAddressesCaptor2.getValue().getAddresses()).containsExactly(eag21); - ArgumentCaptor resolvedAddressesCaptor3 = - ArgumentCaptor.forClass(ResolvedAddresses.class); - verify(loadBalancers.get("sz3")).handleResolvedAddresses(resolvedAddressesCaptor3.capture()); - assertThat(resolvedAddressesCaptor3.getValue().getAddresses()).containsExactly(eag31); - verify(helper, times(2)).updateBalancingState(CONNECTING, BUFFER_PICKER); - - // update with all endpoints unhealthy - lbEndpoint11 = new LbEndpoint(eag11, 11, false); // unhealthy - lbEndpoint12 = new LbEndpoint(eag12, 12, false); // unhealthy - lbEndpoint31 = new LbEndpoint(eag31, 31, false); // unhealthy - lbEndpoint32 = new LbEndpoint(eag32, 32, false); // unhealthy - localityInfo1 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint11, lbEndpoint12), 1, 0); - localityInfo3 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint31, lbEndpoint32), 3, 0); - localityInfoMap = ImmutableMap.of(locality1, localityInfo1, locality3, localityInfo3); - localityStore.updateLocalityStore(localityInfoMap); - - verify(loadBalancers.get("sz1"), times(2)).handleNameResolutionError(statusCaptor.capture()); - assertThat(statusCaptor.getValue().getCode()).isEqualTo(Status.Code.UNAVAILABLE); - verify(loadBalancers.get("sz3"), times(2)).handleNameResolutionError(statusCaptor.capture()); - assertThat(statusCaptor.getValue().getCode()).isEqualTo(Status.Code.UNAVAILABLE); - - // mimic child balancers update subchannels to TRANSIENT_FAILURE after handleNameResolutionError - childHelpers.get("sz1").updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(UNAVAILABLE)); - childHelpers.get("sz3").updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(UNAVAILABLE)); - verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); - } - - @Test - public void updateLocalityStore_OnlyUpdatingWeightsStillUpdatesPicker() { - LocalityLbEndpoints localityInfo1 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint11, lbEndpoint12), 1, 0); - LocalityLbEndpoints localityInfo2 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint21, lbEndpoint22), 2, 0); - LocalityLbEndpoints localityInfo3 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint31, lbEndpoint32), 3, 0); - ImmutableMap localityInfoMap = ImmutableMap.of( - locality1, localityInfo1, locality2, localityInfo2, locality3, localityInfo3); - localityStore.updateLocalityStore(localityInfoMap); - - assertThat(loadBalancers).hasSize(3); - assertThat(loadBalancers.keySet()).containsExactly("sz1", "sz2", "sz3"); - - // Update locality weights before any subchannel becomes READY. - localityInfo1 = new LocalityLbEndpoints(ImmutableList.of(lbEndpoint11, lbEndpoint12), 4, 0); - localityInfo2 = new LocalityLbEndpoints(ImmutableList.of(lbEndpoint21, lbEndpoint22), 5, 0); - localityInfo3 = new LocalityLbEndpoints(ImmutableList.of(lbEndpoint31, lbEndpoint32), 6, 0); - localityInfoMap = ImmutableMap.of( - locality1, localityInfo1, locality2, localityInfo2, locality3, localityInfo3); - localityStore.updateLocalityStore(localityInfoMap); - - final Map localitiesBySubchannel = new HashMap<>(); - for (final Helper h : childHelpers.values()) { - h.updateBalancingState(READY, new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - Subchannel subchannel = mock(Subchannel.class); - localitiesBySubchannel.put(subchannel, namedLocalities.get(h.getAuthority())); - return PickResult.withSubchannel(subchannel); - } - }); - } - - ArgumentCaptor subchannelPickerCaptor = - ArgumentCaptor.forClass(SubchannelPicker.class); - verify(helper, atLeastOnce()).updateBalancingState( - same(READY), subchannelPickerCaptor.capture()); - WeightedRandomPicker interLocalityPicker = - (WeightedRandomPicker) subchannelPickerCaptor.getValue(); - assertThat(interLocalityPicker.weightedChildPickers).hasSize(3); - for (WeightedChildPicker weightedChildPicker : interLocalityPicker.weightedChildPickers) { - Subchannel subchannel - = weightedChildPicker.getPicker().pickSubchannel(pickSubchannelArgs).getSubchannel(); - assertThat(weightedChildPicker.getWeight()) - .isEqualTo( - localityInfoMap.get(localitiesBySubchannel.get(subchannel)).getLocalityWeight()); - } - } - - @Test - public void updateLocalityStore_emptyEndpoints() { - localityStore.updateLocalityStore(Collections.emptyMap()); - assertThat(loadBalancers).hasSize(0); - ArgumentCaptor pickerCaptor = ArgumentCaptor.forClass(null); - verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); - PickResult result = pickerCaptor.getValue().pickSubchannel(mock(PickSubchannelArgs.class)); - assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAVAILABLE); - } - - @Test - public void reset() { - LocalityLbEndpoints localityInfo1 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint11, lbEndpoint12), 1, 0); - LocalityLbEndpoints localityInfo2 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint21, lbEndpoint22), 2, 0); - ImmutableMap localityInfoMap = ImmutableMap.of( - locality1, localityInfo1, locality2, localityInfo2); - localityStore.updateLocalityStore(localityInfoMap); - - assertThat(loadBalancers).hasSize(2); - - localityStore.reset(); - - verify(loadBalancers.get("sz1")).shutdown(); - verify(loadBalancers.get("sz2")).shutdown(); - verify(loadStatsStore).removeLocality(locality1); - verify(loadStatsStore).removeLocality(locality2); - - // Regression test for same locality added back. - localityStore.updateLocalityStore(localityInfoMap); - assertThat(loadBalancers).hasSize(2); - localityStore.reset(); - verify(loadBalancers.get("sz1")).shutdown(); - verify(loadBalancers.get("sz2")).shutdown(); - verify(loadStatsStore, times(2)).removeLocality(locality1); - verify(loadStatsStore, times(2)).removeLocality(locality2); - } - - /** - * Tests the scenario of the following sequence of events. - * (In the format of "event detail - expected state", with abbreviations C for CONNECTING, - * F for TRANSIENT_FAILURE, R for READ, and D for deactivated etc.) - * EDS update: P0 sz1; P1 sz2; P2 sz3; P3 sz4 - P0 C, P1 N/A, P2 N/A, P3 N/A - * 10 secs passes P0 still CONNECTING - P0 C, P1 C, P2 N/A, P3 N/A - * 5 secs passes P1 in TRANSIENT_FAILURE - P0 C, P1 F, P2 C, P3 N/A - * 4 secs passes P2 READY - P0 C, P1 F, P2 R, P3 N/A - * P0 gets READY - P0 R, P1 F&D, P2 R&D, P3 N/A - * P1 gets READY - P0 R, P1 R&D, P2 R&D, P3 N/A - * 10 min passes P0 in TRANSIENT_FAILURE - P0 F, P1 R, P2 R&D, P3 N/A - * 5 min passes - P0 F, P1 R, P2 N/A, P3 N/A - * P1 in TRANSIENT_FAILURE - P0 F, P1 F, P2 C, P3 N/A - * 10 secs passes - P0 F, P1 F, P2 C, P3 C - * P1, P3 gets READY - P0 F, P1 R, P2 C&D, P3 R&D - * EDS update, localities moved: P0 sz1, sz3; P1 sz4; P2 sz2 - P0 C, P1 R, P2 R&D - * 15 min passes - P0 C, P1 R, P2 N/A - * EDS update, locality removed: P0 sz1, sz3, - P0 C, P1 N/A, sz4 R&D - * sz3 gets READY - P0 R, P1 N/A, sz4 R&D - * EDS update, locality comes back and another removed: P0 sz1, P1 sz4 - P0 C, P1 R, sz3 R&D - * - *

Should also verify that when locality store is updated with new EDS data, state of all - * localities should be updated before the child balancer of each locality handles new addresses. - */ - @Test - public void multipriority() { - // EDS update: P0 sz1; P1 sz2; P2 sz3; P3 sz4 - P0 C, P1 N/A, P2 N/A, P3 N/A - LocalityLbEndpoints localityInfo1 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint11, lbEndpoint12), 1, 0); - LocalityLbEndpoints localityInfo2 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint21, lbEndpoint22), 2, 1); - LocalityLbEndpoints localityInfo3 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint31, lbEndpoint32), 3, 2); - LocalityLbEndpoints localityInfo4 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint41, lbEndpoint42), 3, 3); - final ImmutableMap localityInfoMap = ImmutableMap.of( - locality1, localityInfo1, locality2, localityInfo2, locality3, localityInfo3, locality4, - localityInfo4); - syncContext.execute(new Runnable() { - @Override - public void run() { - localityStore.updateLocalityStore(localityInfoMap); - } - }); - assertThat(loadBalancers.keySet()).containsExactly("sz1"); - LoadBalancer lb1 = loadBalancers.get("sz1"); - InOrder inOrder = inOrder(lb1, helper); - ArgumentCaptor resolvedAddressesCaptor1 = ArgumentCaptor.forClass(null); - inOrder.verify(helper).updateBalancingState(CONNECTING, BUFFER_PICKER); - inOrder.verify(lb1).handleResolvedAddresses(resolvedAddressesCaptor1.capture()); - assertThat(resolvedAddressesCaptor1.getValue().getAddresses()).containsExactly(eag11, eag12); - - // 10 sec passes P0 still CONNECTING - P0 C, P1 C, P2 N/A, P3 N/A - fakeClock.forwardTime(9, TimeUnit.SECONDS); - assertThat(loadBalancers.keySet()).containsExactly("sz1"); - assertThat(fakeClock.getPendingTasks(failOverTaskFilter)).hasSize(1); - ScheduledTask failOverTask = - Iterables.getOnlyElement(fakeClock.getPendingTasks(failOverTaskFilter)); - fakeClock.forwardTime(1, TimeUnit.SECONDS); - assertThat(failOverTask.isCancelled()).isFalse(); - assertThat(fakeClock.getPendingTasks(failOverTaskFilter)).doesNotContain(failOverTask); - assertThat(fakeClock.getPendingTasks(failOverTaskFilter)).hasSize(1); - failOverTask = Iterables.getOnlyElement(fakeClock.getPendingTasks(failOverTaskFilter)); - assertThat(fakeClock.getPendingTasks(deactivationTaskFilter)).isEmpty(); - assertThat(loadBalancers.keySet()).containsExactly("sz1", "sz2"); - LoadBalancer lb2 = loadBalancers.get("sz2"); - inOrder = inOrder(lb1, lb2, helper); - inOrder.verify(helper).updateBalancingState(CONNECTING, BUFFER_PICKER); - ArgumentCaptor resolvedAddressesCaptor2 = ArgumentCaptor.forClass(null); - inOrder.verify(lb2).handleResolvedAddresses(resolvedAddressesCaptor2.capture()); - assertThat(resolvedAddressesCaptor2.getValue().getAddresses()).containsExactly(eag21, eag22); - - // 5 secs passes P1 in TRANSIENT_FAILURE - P0 C, P1 F, P2 C, P3 N/A - fakeClock.forwardTime(5, TimeUnit.SECONDS); - assertThat(fakeClock.getPendingTasks(failOverTaskFilter)).containsExactly(failOverTask); - final Helper helper2 = childHelpers.get("sz2"); - helper.getSynchronizationContext().execute(new Runnable() { - @Override - public void run() { - helper2.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(UNAVAILABLE)); - } - }); - assertThat(failOverTask.isCancelled()).isTrue(); - assertThat(fakeClock.getPendingTasks(failOverTaskFilter)).doesNotContain(failOverTask); - assertThat(fakeClock.getPendingTasks(failOverTaskFilter)).hasSize(1); - failOverTask = Iterables.getOnlyElement(fakeClock.getPendingTasks(failOverTaskFilter)); - assertThat(fakeClock.getPendingTasks(deactivationTaskFilter)).isEmpty(); - assertThat(loadBalancers.keySet()).containsExactly("sz1", "sz2", "sz3"); - LoadBalancer lb3 = loadBalancers.get("sz3"); - inOrder = inOrder(lb3, helper); - // The order of the following two updateBalancingState() does not matter. We want to verify - // lb3.handleResolvedAddresses() is after them. - inOrder.verify(helper).updateBalancingState(same(TRANSIENT_FAILURE), isA(ErrorPicker.class)); - inOrder.verify(helper).updateBalancingState(CONNECTING, BUFFER_PICKER); - ArgumentCaptor resolvedAddressesCaptor3 = ArgumentCaptor.forClass(null); - inOrder.verify(lb3).handleResolvedAddresses(resolvedAddressesCaptor3.capture()); - assertThat(resolvedAddressesCaptor3.getValue().getAddresses()).containsExactly(eag31, eag32); - - // 5 secs passes P2 READY - P0 C, P1 F, P2 R, P3 N/A - fakeClock.forwardTime(4, TimeUnit.SECONDS); - assertThat(fakeClock.getPendingTasks(failOverTaskFilter)).containsExactly(failOverTask); - final Helper helper3 = childHelpers.get("sz3"); - final Subchannel mockSubchannel3 = mock(Subchannel.class); - helper.getSynchronizationContext().execute(new Runnable() { - @Override - public void run() { - helper3.updateBalancingState( - READY, - new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withSubchannel(mockSubchannel3); - } - }); - } - }); - assertThat(failOverTask.isCancelled()).isTrue(); - assertThat(fakeClock.getPendingTasks(failOverTaskFilter)).isEmpty(); - assertThat(fakeClock.getPendingTasks(deactivationTaskFilter)).isEmpty(); - ArgumentCaptor subchannelPickerCaptor = ArgumentCaptor.forClass(null); - inOrder.verify(helper).updateBalancingState(same(READY), subchannelPickerCaptor.capture()); - assertThat(subchannelPickerCaptor.getValue() - .pickSubchannel(mock(PickSubchannelArgs.class)) - .getSubchannel()) - .isSameInstanceAs(mockSubchannel3); - - // P0 gets READY - P0 R, P1 F&D, P2 R&D, P3 N/A - final Helper helper1 = childHelpers.get("sz1"); - final Subchannel mockSubchannel1 = mock(Subchannel.class); - helper.getSynchronizationContext().execute(new Runnable() { - @Override - public void run() { - helper1.updateBalancingState( - READY, - new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withSubchannel(mockSubchannel1); - } - }); - } - }); - assertThat(fakeClock.getPendingTasks(failOverTaskFilter)).isEmpty(); - Collection deactivationTasks = fakeClock.getPendingTasks(deactivationTaskFilter); - assertThat(deactivationTasks).hasSize(2); - inOrder.verify(helper).updateBalancingState(same(READY), subchannelPickerCaptor.capture()); - assertThat(subchannelPickerCaptor.getValue() - .pickSubchannel(mock(PickSubchannelArgs.class)) - .getSubchannel()) - .isSameInstanceAs(mockSubchannel1); - - // P1 gets READY - P0 R, P1 R&D, P2 R&D, P3 N/A - final Subchannel mockSubchannel2 = mock(Subchannel.class); - helper.getSynchronizationContext().execute(new Runnable() { - @Override - public void run() { - helper2.updateBalancingState( - READY, - new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withSubchannel(mockSubchannel2); - } - }); - } - }); - assertThat(fakeClock.getPendingTasks(failOverTaskFilter)).isEmpty(); - assertThat(fakeClock.getPendingTasks(deactivationTaskFilter)).hasSize(2); - inOrder.verify(helper, never()) - .updateBalancingState(any(ConnectivityState.class), any(SubchannelPicker.class)); - - // 10 min passes P0 in TRANSIENT_FAILURE - P0 F, P1 R, P2 R&D, P3 N/A - fakeClock.forwardTime(10, TimeUnit.MINUTES); - helper.getSynchronizationContext().execute(new Runnable() { - @Override - public void run() { - helper1.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(UNAVAILABLE)); - } - }); - assertThat(fakeClock.getPendingTasks(failOverTaskFilter)).isEmpty(); - assertThat(fakeClock.getPendingTasks(deactivationTaskFilter)).hasSize(1); - ScheduledTask deactivationTask = - Iterables.getOnlyElement(fakeClock.getPendingTasks(deactivationTaskFilter)); - assertThat(deactivationTask).isSameInstanceAs(Iterables.get(deactivationTasks, 1)); - assertThat(Iterables.get(deactivationTasks, 0).isCancelled()).isTrue(); - inOrder.verify(helper, never()) - .updateBalancingState(CONNECTING, BUFFER_PICKER); - - - // 5 min passes - P0 F, P1 R, P2 N/A, P3 N/A - verify(lb3, never()).shutdown(); - fakeClock.forwardTime(5, TimeUnit.MINUTES); - assertThat(fakeClock.getPendingTasks(failOverTaskFilter)).isEmpty(); - assertThat(fakeClock.getPendingTasks(deactivationTaskFilter)).isEmpty(); - verify(lb3).shutdown(); - - // P1 in TRANSIENT_FAILURE - P0 F, P1 F, P2 C, P3 N/A - helper.getSynchronizationContext().execute(new Runnable() { - @Override - public void run() { - helper2.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(UNAVAILABLE)); - } - }); - assertThat(fakeClock.getPendingTasks(failOverTaskFilter)).hasSize(1); - failOverTask = Iterables.getOnlyElement(fakeClock.getPendingTasks(failOverTaskFilter)); - assertThat(fakeClock.getPendingTasks(deactivationTaskFilter)).isEmpty(); - // The order of the following two updateBalancingState() does not matter. We only want to verify - // these are the latest tow updateBalancingState(). - inOrder.verify(helper).updateBalancingState(same(TRANSIENT_FAILURE), isA(ErrorPicker.class)); - inOrder.verify(helper).updateBalancingState(CONNECTING, BUFFER_PICKER); - assertThat(loadBalancers.keySet()).containsExactly("sz1", "sz2", "sz3"); - assertThat(loadBalancers.get("sz3")).isNotSameInstanceAs(lb3); - lb3 = loadBalancers.get("sz3"); - assertThat(childHelpers.get("sz3")).isNotSameInstanceAs(helper3); - - // 10 secs passes - P0 F, P1 F, P2 C, P3 C - fakeClock.forwardTime(10, TimeUnit.SECONDS); - assertThat(fakeClock.getPendingTasks(failOverTaskFilter)).hasSize(1); - assertThat(fakeClock.getPendingTasks(failOverTaskFilter)).doesNotContain(failOverTask); - assertThat(fakeClock.getPendingTasks(deactivationTaskFilter)).isEmpty(); - assertThat(loadBalancers.keySet()).containsExactly("sz1", "sz2", "sz3", "sz4"); - LoadBalancer lb4 = loadBalancers.get("sz4"); - inOrder = inOrder(lb1, lb2, lb3, lb4, helper); - inOrder.verify(helper).updateBalancingState(CONNECTING, BUFFER_PICKER); - ArgumentCaptor resolvedAddressesCaptor4 = ArgumentCaptor.forClass(null); - inOrder.verify(lb4).handleResolvedAddresses(resolvedAddressesCaptor4.capture()); - assertThat(resolvedAddressesCaptor4.getValue().getAddresses()).containsExactly(eag41, eag42); - - // P1, P3 gets READY - P0 F, P1 R, P2 C&D, P3 R&D - final Subchannel mockSubchannel22 = mock(Subchannel.class); - final Subchannel mockSubchannel4 = mock(Subchannel.class); - final Helper helper4 = childHelpers.get("sz4"); - helper.getSynchronizationContext().execute(new Runnable() { - @Override - public void run() { - helper2.updateBalancingState( - READY, - new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withSubchannel(mockSubchannel22); - } - }); - helper4.updateBalancingState( - READY, - new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withSubchannel(mockSubchannel4); - } - }); - } - }); - assertThat(fakeClock.getPendingTasks(failOverTaskFilter)).isEmpty(); - assertThat(fakeClock.getPendingTasks(deactivationTaskFilter)).hasSize(2); - inOrder.verify(helper).updateBalancingState(same(READY), subchannelPickerCaptor.capture()); - assertThat(subchannelPickerCaptor.getValue() - .pickSubchannel(mock(PickSubchannelArgs.class)) - .getSubchannel()) - .isSameInstanceAs(mockSubchannel22); - - // EDS update, localities moved: P0 sz1, sz3; P1 sz4; P2 sz2 - P0 C, P1 R, P2 R&D - localityInfo1 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint12), 1, 0); - localityInfo2 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint22), 2, 2); - localityInfo3 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint32), 3, 0); - localityInfo4 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint42), 4, 1); - final ImmutableMap localityInfoMap2 = ImmutableMap.of( - locality1, localityInfo1, locality2, localityInfo2, locality3, localityInfo3, locality4, - localityInfo4); - syncContext.execute(new Runnable() { - @Override - public void run() { - localityStore.updateLocalityStore(localityInfoMap2); - } - }); - assertThat(loadBalancers.keySet()).containsExactly("sz1", "sz2", "sz3", "sz4"); - assertThat(loadBalancers.values()).containsExactly(lb1, lb2, lb3, lb4); - inOrder.verify(helper).updateBalancingState(same(READY), subchannelPickerCaptor.capture()); - assertThat(subchannelPickerCaptor.getValue() - .pickSubchannel(mock(PickSubchannelArgs.class)) - .getSubchannel()) - .isSameInstanceAs(mockSubchannel4); - // The order of the following four handleResolvedAddresses() does not matter. We want to verify - // they are after helper.updateBalancingState() - inOrder.verify(lb1).handleResolvedAddresses(resolvedAddressesCaptor1.capture()); - assertThat(resolvedAddressesCaptor1.getValue().getAddresses()).containsExactly(eag12); - inOrder.verify(lb2).handleResolvedAddresses(resolvedAddressesCaptor2.capture()); - assertThat(resolvedAddressesCaptor2.getValue().getAddresses()).containsExactly(eag22); - inOrder.verify(lb3).handleResolvedAddresses(resolvedAddressesCaptor3.capture()); - assertThat(resolvedAddressesCaptor3.getValue().getAddresses()).containsExactly(eag32); - inOrder.verify(lb4).handleResolvedAddresses(resolvedAddressesCaptor4.capture()); - assertThat(resolvedAddressesCaptor4.getValue().getAddresses()).containsExactly(eag42); - assertThat(fakeClock.getPendingTasks(failOverTaskFilter)).isEmpty(); - assertThat(fakeClock.getPendingTasks(deactivationTaskFilter)).hasSize(1); - - // 15 min passes - P0 C, P1 R, P2 N/A - verify(lb2, never()).shutdown(); - fakeClock.forwardTime(15, TimeUnit.MINUTES); - assertThat(fakeClock.getPendingTasks(failOverTaskFilter)).isEmpty(); - assertThat(fakeClock.getPendingTasks(deactivationTaskFilter)).isEmpty(); - inOrder.verify(lb2).shutdown(); - inOrder.verifyNoMoreInteractions(); - - // EDS update, locality removed: P0 sz1, sz3, - P0 C, P1 N/A, sz4 R&D - localityInfo1 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint11), 1, 0); - localityInfo3 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint31), 3, 0); - final ImmutableMap localityInfoMap3 = ImmutableMap.of( - locality1, localityInfo1,locality3, localityInfo3); - syncContext.execute(new Runnable() { - @Override - public void run() { - localityStore.updateLocalityStore(localityInfoMap3); - } - }); - inOrder.verify(helper).updateBalancingState(CONNECTING, BUFFER_PICKER); - // The order of the following two handleResolvedAddresses() does not matter. We want to verify - // they are after helper.updateBalancingState() - inOrder.verify(lb1).handleResolvedAddresses(resolvedAddressesCaptor1.capture()); - assertThat(resolvedAddressesCaptor1.getValue().getAddresses()).containsExactly(eag11); - inOrder.verify(lb3).handleResolvedAddresses(resolvedAddressesCaptor3.capture()); - assertThat(resolvedAddressesCaptor3.getValue().getAddresses()).containsExactly(eag31); - inOrder.verifyNoMoreInteractions(); - assertThat(fakeClock.getPendingTasks(failOverTaskFilter)).isEmpty(); - assertThat(fakeClock.getPendingTasks(deactivationTaskFilter)).hasSize(1); - - // sz3 gets READY - P0 R, P1 N/A, sz4 R&D - final Helper newHelper3 = childHelpers.get("sz3"); - final Subchannel newMockSubchannel3 = mock(Subchannel.class); - syncContext.execute(new Runnable() { - @Override - public void run() { - newHelper3.updateBalancingState( - READY, - new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withSubchannel(newMockSubchannel3); - } - } - ); - } - }); - assertThat(fakeClock.getPendingTasks(failOverTaskFilter)).isEmpty(); - assertThat(fakeClock.getPendingTasks(deactivationTaskFilter)).hasSize(1); - inOrder.verify(helper).updateBalancingState(same(READY), subchannelPickerCaptor.capture()); - assertThat(subchannelPickerCaptor.getValue() - .pickSubchannel(mock(PickSubchannelArgs.class)) - .getSubchannel()) - .isSameInstanceAs(newMockSubchannel3); - inOrder.verifyNoMoreInteractions(); - - // EDS update, locality comes back and another removed: P0 sz1, P1 sz4 - P0 C, P1 R, sz3 R&D - localityInfo1 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint11), 1, 0); - localityInfo4 = - new LocalityLbEndpoints(ImmutableList.of(lbEndpoint41), 4, 1); - final ImmutableMap localityInfoMap4 = ImmutableMap.of( - locality1, localityInfo1,locality4, localityInfo4); - syncContext.execute(new Runnable() { - @Override - public void run() { - localityStore.updateLocalityStore(localityInfoMap4); - } - }); - inOrder.verify(helper, atLeastOnce()).updateBalancingState( - same(READY), subchannelPickerCaptor.capture()); - assertThat(subchannelPickerCaptor.getValue() - .pickSubchannel(mock(PickSubchannelArgs.class)) - .getSubchannel()) - .isSameInstanceAs(mockSubchannel4); - // The order of the following two handleResolvedAddresses() does not matter. We want to verify - // they are after helper.updateBalancingState() - inOrder.verify(lb1).handleResolvedAddresses(resolvedAddressesCaptor1.capture()); - assertThat(resolvedAddressesCaptor1.getValue().getAddresses()).containsExactly(eag11); - inOrder.verify(lb4).handleResolvedAddresses(resolvedAddressesCaptor4.capture()); - assertThat(resolvedAddressesCaptor4.getValue().getAddresses()).containsExactly(eag41); - inOrder.verifyNoMoreInteractions(); - assertThat(fakeClock.getPendingTasks(failOverTaskFilter)).isEmpty(); - assertThat(fakeClock.getPendingTasks(deactivationTaskFilter)).hasSize(1); - - verify(lb1, never()).shutdown(); - verify(lb3, never()).shutdown(); - verify(lb4, never()).shutdown(); - localityStore.reset(); - assertThat(fakeClock.getPendingTasks(deactivationTaskFilter)).isEmpty(); - verify(lb1).shutdown(); - verify(lb3).shutdown(); - verify(lb4).shutdown(); - } - - private static final class FakeLoadStatsStore implements LoadStatsStore { - - Map localityCounters = new HashMap<>(); - - @Override - public ClusterStats generateLoadReport() { - throw new AssertionError("Should not be called"); - } - - @Override - public ClientLoadCounter addLocality(Locality locality) { - assertThat(localityCounters).doesNotContainKey(locality); - ClientLoadCounter counter = new ClientLoadCounter(); - localityCounters.put(locality, counter); - return counter; - } - - @Override - public void removeLocality(Locality locality) { - assertThat(localityCounters).containsKey(locality); - localityCounters.remove(locality); - } - - @Override - public void recordDroppedRequest(String category) { - // NO-OP, verify by invocations. - } - } - - private static final class FakeOrcaReportingHelperWrapper extends OrcaReportingHelperWrapper { - - final Helper delegate; - long reportIntervalNanos = -1; - - FakeOrcaReportingHelperWrapper(Helper delegate) { - this.delegate = checkNotNull(delegate, "delegate"); - } - - @Override - public void setReportingConfig(OrcaReportingConfig config) { - reportIntervalNanos = config.getReportIntervalNanos(); - } - - @Override - public Helper asHelper() { - return delegate; - } - } -} diff --git a/xds/src/test/java/io/grpc/xds/XdsRoutingLoadBalancerProviderTest.java b/xds/src/test/java/io/grpc/xds/XdsRoutingLoadBalancerProviderTest.java deleted file mode 100644 index ec8c73fc722..00000000000 --- a/xds/src/test/java/io/grpc/xds/XdsRoutingLoadBalancerProviderTest.java +++ /dev/null @@ -1,231 +0,0 @@ -/* - * Copyright 2020 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.xds; - -import static com.google.common.truth.Truth.assertThat; -import static org.mockito.Mockito.mock; - -import com.google.re2j.Pattern; -import io.grpc.LoadBalancer; -import io.grpc.LoadBalancer.Helper; -import io.grpc.LoadBalancerProvider; -import io.grpc.LoadBalancerRegistry; -import io.grpc.NameResolver.ConfigOrError; -import io.grpc.internal.JsonParser; -import io.grpc.internal.ServiceConfigUtil.PolicySelection; -import io.grpc.xds.RouteMatch.FractionMatcher; -import io.grpc.xds.RouteMatch.HeaderMatcher; -import io.grpc.xds.RouteMatch.PathMatcher; -import io.grpc.xds.XdsRoutingLoadBalancerProvider.Route; -import io.grpc.xds.XdsRoutingLoadBalancerProvider.XdsRoutingConfig; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Tests for {@link XdsRoutingLoadBalancerProvider}. */ -@RunWith(JUnit4.class) -public class XdsRoutingLoadBalancerProviderTest { - - @Test - public void parseXdsRoutingLoadBalancingPolicyConfig() throws Exception { - LoadBalancerRegistry lbRegistry = new LoadBalancerRegistry(); - XdsRoutingLoadBalancerProvider xdsRoutingLoadBalancerProvider = - new XdsRoutingLoadBalancerProvider(lbRegistry); - final Object fooConfig = new Object(); - LoadBalancerProvider lbProviderFoo = new LoadBalancerProvider() { - @Override - public boolean isAvailable() { - return true; - } - - @Override - public int getPriority() { - return 5; - } - - @Override - public String getPolicyName() { - return "foo_policy"; - } - - @Override - public LoadBalancer newLoadBalancer(Helper helper) { - return mock(LoadBalancer.class); - } - - @Override - public ConfigOrError parseLoadBalancingPolicyConfig(Map rawConfig) { - return ConfigOrError.fromConfig(fooConfig); - } - }; - final Object barConfig = new Object(); - LoadBalancerProvider lbProviderBar = new LoadBalancerProvider() { - @Override - public boolean isAvailable() { - return true; - } - - @Override - public int getPriority() { - return 5; - } - - @Override - public String getPolicyName() { - return "bar_policy"; - } - - @Override - public LoadBalancer newLoadBalancer(Helper helper) { - return mock(LoadBalancer.class); - } - - @Override - public ConfigOrError parseLoadBalancingPolicyConfig(Map rawConfig) { - return ConfigOrError.fromConfig(barConfig); - } - }; - lbRegistry.register(lbProviderFoo); - lbRegistry.register(lbProviderBar); - - String xdsRoutingConfigJson = ("{\n" - + " 'route' : [\n" - + " {\n" - + " 'path' : '/service_1/method_1',\n" - + " 'action' : 'action_foo'\n" - + " },\n" - + " {\n" - + " 'path' : '/service_1/method_2',\n" - + " 'headers' : [\n" - + " {\n" - + " 'name' : ':scheme',\n" - + " 'exactMatch' : 'https'\n" - + " }\n" - + " ],\n" - + " 'action' : 'action_bar'\n" - + " },\n" - + " {\n" - + " 'prefix' : '/service_2/',\n" - + " 'headers' : [\n" - + " {\n" - + " 'name' : ':path',\n" - + " 'regexMatch' : 'google.*'\n" - + " }\n" - + " ],\n" - + " 'matchFraction' : {\n" - + " 'numerator' : 10,\n" - + " 'denominator' : 100\n" - + " },\n" - + " 'action' : 'action_bar'\n" - + " },\n" - + " {\n" - + " 'regex' : '^/service_2/method_3$',\n" - + " 'headers' : [\n" - + " {\n" - + " 'name' : ':method',\n" - + " 'presentMatch' : true,\n" - + " 'invertMatch' : true\n" - + " },\n" - + " {\n" - + " 'name' : 'timeout',\n" - + " 'rangeMatch' : {\n" - + " 'start' : 0,\n" - + " 'end' : 10\n" - + " }\n" - + " }\n" - + " ],\n" - + " 'matchFraction' : {\n" - + " 'numerator' : 55,\n" - + " 'denominator' : 1000\n" - + " },\n" - + " 'action' : 'action_foo'\n" - + " }\n" - + " ],\n" - + " 'action' : {\n" - + " 'action_foo' : {\n" - + " 'childPolicy' : [\n" - + " {'unsupported_policy' : {}},\n" - + " {'foo_policy' : {}}\n" - + " ]\n" - + " },\n" - + " 'action_bar' : {\n" - + " 'childPolicy' : [\n" - + " {'unsupported_policy' : {}},\n" - + " {'bar_policy' : {}}\n" - + " ]\n" - + " }\n" - + " }\n" - + "}\n").replace("'", "\""); - - @SuppressWarnings("unchecked") - Map rawLbConfigMap = (Map) JsonParser.parse(xdsRoutingConfigJson); - ConfigOrError configOrError = - xdsRoutingLoadBalancerProvider.parseLoadBalancingPolicyConfig(rawLbConfigMap); - assertThat(configOrError.getConfig()).isNotNull(); - XdsRoutingConfig config = (XdsRoutingConfig) configOrError.getConfig(); - List configRoutes = config.routes; - assertThat(configRoutes).hasSize(4); - assertThat(configRoutes.get(0)).isEqualTo( - new Route( - new RouteMatch( - new PathMatcher("/service_1/method_1", null, null), - Collections.emptyList(), null), - "action_foo")); - assertThat(configRoutes.get(1)).isEqualTo( - new Route( - new RouteMatch( - new PathMatcher("/service_1/method_2", null, null), - Arrays.asList( - new HeaderMatcher(":scheme", "https", null, null, null, null, - null, false)), - null), - "action_bar")); - assertThat(configRoutes.get(2)).isEqualTo( - new Route( - new RouteMatch( - new PathMatcher(null, "/service_2/", null), - Arrays.asList( - new HeaderMatcher(":path", null, Pattern.compile("google.*"), null, - null, null, null, false)), - new FractionMatcher(10, 100)), - "action_bar")); - assertThat(configRoutes.get(3)).isEqualTo( - new Route( - new RouteMatch( - new PathMatcher(null, null, Pattern.compile("^/service_2/method_3$")), - Arrays.asList( - new HeaderMatcher(":method", null, null, null, - true, null, null, true), - new HeaderMatcher("timeout", null, null, - new HeaderMatcher.Range(0, 10), null, null, null, false)), - new FractionMatcher(55, 1000)), - "action_foo")); - - Map configActions = config.actions; - assertThat(configActions).hasSize(2); - assertThat(configActions).containsExactly( - "action_foo", - new PolicySelection(lbProviderFoo, fooConfig), - "action_bar", - new PolicySelection( - lbProviderBar, barConfig)); - } -} diff --git a/xds/src/test/java/io/grpc/xds/XdsRoutingLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/XdsRoutingLoadBalancerTest.java deleted file mode 100644 index 10b11398bc8..00000000000 --- a/xds/src/test/java/io/grpc/xds/XdsRoutingLoadBalancerTest.java +++ /dev/null @@ -1,478 +0,0 @@ -/* - * Copyright 2020 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.xds; - -import static com.google.common.truth.Truth.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.atLeast; -import static org.mockito.Mockito.atLeastOnce; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; -import io.grpc.CallOptions; -import io.grpc.ConnectivityState; -import io.grpc.EquivalentAddressGroup; -import io.grpc.LoadBalancer; -import io.grpc.LoadBalancer.Helper; -import io.grpc.LoadBalancer.PickResult; -import io.grpc.LoadBalancer.PickSubchannelArgs; -import io.grpc.LoadBalancer.ResolvedAddresses; -import io.grpc.LoadBalancer.Subchannel; -import io.grpc.LoadBalancer.SubchannelPicker; -import io.grpc.LoadBalancerProvider; -import io.grpc.Metadata; -import io.grpc.MethodDescriptor; -import io.grpc.MethodDescriptor.MethodType; -import io.grpc.Status; -import io.grpc.Status.Code; -import io.grpc.SynchronizationContext; -import io.grpc.internal.FakeClock; -import io.grpc.internal.PickSubchannelArgsImpl; -import io.grpc.internal.ServiceConfigUtil.PolicySelection; -import io.grpc.testing.TestMethodDescriptors; -import io.grpc.xds.RouteMatch.HeaderMatcher; -import io.grpc.xds.RouteMatch.PathMatcher; -import io.grpc.xds.XdsRoutingLoadBalancer.RouteMatchingSubchannelPicker; -import io.grpc.xds.XdsRoutingLoadBalancerProvider.Route; -import io.grpc.xds.XdsRoutingLoadBalancerProvider.XdsRoutingConfig; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; - -/** Tests for {@link XdsRoutingLoadBalancer}. */ -@RunWith(JUnit4.class) -public class XdsRoutingLoadBalancerTest { - - private final SynchronizationContext syncContext = new SynchronizationContext( - new Thread.UncaughtExceptionHandler() { - @Override - public void uncaughtException(Thread t, Throwable e) { - throw new AssertionError(e); - } - }); - private final FakeClock fakeClock = new FakeClock(); - - @Mock - private LoadBalancer.Helper helper; - @Captor - ArgumentCaptor pickerCaptor; - - private RouteMatch routeMatch1 = - new RouteMatch( - new PathMatcher("/FooService/barMethod", null, null), - Arrays.asList( - new HeaderMatcher("user-agent", "gRPC-Java", null, null, null, null, null, false), - new HeaderMatcher("grpc-encoding", "gzip", null, null, null, null, null, false)), - null); - private RouteMatch routeMatch2 = - new RouteMatch( - new PathMatcher("/FooService/bazMethod", null, null), - Collections.emptyList(), - null); - private RouteMatch routeMatch3 = - new RouteMatch( - new PathMatcher(null, "/", null), - Collections.emptyList(), - null); - private final Map lbConfigInventory = new HashMap<>(); - private final List childBalancers = new ArrayList<>(); - private LoadBalancer xdsRoutingLoadBalancer; - - @Before - public void setUp() { - MockitoAnnotations.initMocks(this); - when(helper.getSynchronizationContext()).thenReturn(syncContext); - when(helper.getScheduledExecutorService()).thenReturn(fakeClock.getScheduledExecutorService()); - lbConfigInventory.put("actionA", new Object()); - lbConfigInventory.put("actionB", new Object()); - lbConfigInventory.put("actionC", null); - xdsRoutingLoadBalancer = new XdsRoutingLoadBalancer(helper); - } - - @After - public void tearDown() { - xdsRoutingLoadBalancer.shutdown(); - for (FakeLoadBalancer childLb : childBalancers) { - assertThat(childLb.shutdown).isTrue(); - } - } - - @Test - public void handleResolvedAddressesUpdatesChannelPicker() { - deliverResolvedAddresses( - ImmutableMap.of( - new Route(routeMatch1, "actionA"), "policy_a", - new Route(routeMatch2, "actionB"), "policy_b")); - - verify(helper, atLeastOnce()).updateBalancingState( - eq(ConnectivityState.CONNECTING), pickerCaptor.capture()); - RouteMatchingSubchannelPicker picker = (RouteMatchingSubchannelPicker) pickerCaptor.getValue(); - assertThat(picker.routePickers).hasSize(2); - assertThat(picker.routePickers.get(routeMatch1).pickSubchannel(mock(PickSubchannelArgs.class))) - .isEqualTo(PickResult.withNoResult()); - assertThat(picker.routePickers.get(routeMatch2).pickSubchannel(mock(PickSubchannelArgs.class))) - .isEqualTo(PickResult.withNoResult()); - assertThat(childBalancers).hasSize(2); - FakeLoadBalancer childBalancer1 = childBalancers.get(0); - FakeLoadBalancer childBalancer2 = childBalancers.get(1); - assertThat(childBalancer1.name).isEqualTo("policy_a"); - assertThat(childBalancer2.name).isEqualTo("policy_b"); - assertThat(childBalancer1.config).isEqualTo(lbConfigInventory.get("actionA")); - assertThat(childBalancer2.config).isEqualTo(lbConfigInventory.get("actionB")); - - // Receive an updated config. - deliverResolvedAddresses( - ImmutableMap.of( - new Route(routeMatch1, "actionA"), "policy_a", - new Route(routeMatch3, "actionC"), "policy_c")); - - verify(helper, atLeast(2)) - .updateBalancingState(eq(ConnectivityState.CONNECTING), pickerCaptor.capture()); - picker = (RouteMatchingSubchannelPicker) pickerCaptor.getValue(); - assertThat(picker.routePickers).hasSize(2); - assertThat(picker.routePickers).doesNotContainKey(routeMatch2); - assertThat(picker.routePickers.get(routeMatch3).pickSubchannel(mock(PickSubchannelArgs.class))) - .isEqualTo(PickResult.withNoResult()); - assertThat(fakeClock.numPendingTasks()) - .isEqualTo(1); // (delayed) shutdown because "actionB" is removed - assertThat(childBalancer1.shutdown).isFalse(); - assertThat(childBalancer2.shutdown).isFalse(); - - assertThat(childBalancers).hasSize(3); - FakeLoadBalancer childBalancer3 = childBalancers.get(2); - assertThat(childBalancer3.name).isEqualTo("policy_c"); - assertThat(childBalancer3.config).isEqualTo(lbConfigInventory.get("actionC")); - - fakeClock.forwardTime( - XdsRoutingLoadBalancer.DELAYED_ACTION_DELETION_TIME_MINUTES, TimeUnit.MINUTES); - assertThat(childBalancer2.shutdown).isTrue(); - } - - @Test - public void updateWithActionPolicyChange() { - deliverResolvedAddresses(ImmutableMap.of(new Route(routeMatch1, "actionA"), "policy_a")); - FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); - assertThat(childBalancer.name).isEqualTo("policy_a"); - assertThat(childBalancer.config).isEqualTo(lbConfigInventory.get("actionA")); - - deliverResolvedAddresses(ImmutableMap.of(new Route(routeMatch1, "actionA"), "policy_b")); - assertThat(childBalancer.shutdown).isTrue(); // immediate shutdown as the it was not ready - assertThat(Iterables.getOnlyElement(childBalancers).name).isEqualTo("policy_b"); - assertThat(Iterables.getOnlyElement(childBalancers).config) - .isEqualTo(lbConfigInventory.get("actionA")); - } - - @Test - public void updateBalancingStateFromChildBalancers() { - deliverResolvedAddresses( - ImmutableMap.of( - new Route(routeMatch1, "actionA"), "policy_a", - new Route(routeMatch2, "actionB"), "policy_b")); - - assertThat(childBalancers).hasSize(2); - FakeLoadBalancer childBalancer1 = childBalancers.get(0); - FakeLoadBalancer childBalancer2 = childBalancers.get(1); - Subchannel subchannel1 = mock(Subchannel.class); - Subchannel subchannel2 = mock(Subchannel.class); - childBalancer1.deliverSubchannelState(subchannel1, ConnectivityState.READY); - - verify(helper).updateBalancingState(eq(ConnectivityState.READY), pickerCaptor.capture()); - RouteMatchingSubchannelPicker picker = (RouteMatchingSubchannelPicker) pickerCaptor.getValue(); - assertThat(picker.routePickers).hasSize(2); - assertThat( - picker.routePickers.get(routeMatch1) - .pickSubchannel(mock(PickSubchannelArgs.class)).getSubchannel()) - .isEqualTo(subchannel1); - assertThat(picker.routePickers.get(routeMatch2).pickSubchannel(mock(PickSubchannelArgs.class))) - .isEqualTo(PickResult.withNoResult()); - - childBalancer2.deliverSubchannelState(subchannel2, ConnectivityState.READY); - verify(helper, times(2)) - .updateBalancingState(eq(ConnectivityState.READY), pickerCaptor.capture()); - picker = (RouteMatchingSubchannelPicker) pickerCaptor.getValue(); - assertThat( - picker.routePickers.get(routeMatch2) - .pickSubchannel(mock(PickSubchannelArgs.class)).getSubchannel()) - .isEqualTo(subchannel2); - } - - @Test - public void updateBalancingStateFromDeactivatedChildBalancer() { - FakeLoadBalancer balancer = - deliverAddressesAndUpdateToRemoveChildPolicy( - new Route(routeMatch1, "actionA"), "policy_a"); - Subchannel subchannel = mock(Subchannel.class); - balancer.deliverSubchannelState(subchannel, ConnectivityState.READY); - verify(helper, never()).updateBalancingState( - eq(ConnectivityState.READY), any(SubchannelPicker.class)); - - deliverResolvedAddresses(ImmutableMap.of(new Route(routeMatch1, "actionA"), "policy_a")); - verify(helper).updateBalancingState(eq(ConnectivityState.READY), pickerCaptor.capture()); - RouteMatchingSubchannelPicker picker = (RouteMatchingSubchannelPicker) pickerCaptor.getValue(); - assertThat( - picker.routePickers.get(routeMatch1) - .pickSubchannel(mock(PickSubchannelArgs.class)).getSubchannel()) - .isEqualTo(subchannel); - } - - @Test - public void errorPropagation() { - Status error = Status.UNAVAILABLE.withDescription("resolver error"); - xdsRoutingLoadBalancer.handleNameResolutionError(error); - verify(helper).updateBalancingState( - eq(ConnectivityState.TRANSIENT_FAILURE), pickerCaptor.capture()); - PickResult result = pickerCaptor.getValue().pickSubchannel(mock(PickSubchannelArgs.class)); - assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAVAILABLE); - assertThat(result.getStatus().getDescription()).isEqualTo("resolver error"); - - deliverResolvedAddresses( - ImmutableMap.of( - new Route(routeMatch1, "actionA"), "policy_a", - new Route(routeMatch2, "actionB"), "policy_b")); - - assertThat(childBalancers).hasSize(2); - FakeLoadBalancer childBalancer1 = childBalancers.get(0); - FakeLoadBalancer childBalancer2 = childBalancers.get(1); - - xdsRoutingLoadBalancer.handleNameResolutionError(error); - assertThat(childBalancer1.upstreamError.getCode()).isEqualTo(Code.UNAVAILABLE); - assertThat(childBalancer1.upstreamError.getDescription()).isEqualTo("resolver error"); - assertThat(childBalancer2.upstreamError.getCode()).isEqualTo(Code.UNAVAILABLE); - assertThat(childBalancer2.upstreamError.getDescription()).isEqualTo("resolver error"); - } - - @Test - public void errorPropagationToDeactivatedChildBalancer() { - FakeLoadBalancer balancer = - deliverAddressesAndUpdateToRemoveChildPolicy( - new Route(routeMatch1, "actionA"), "policy_a"); - xdsRoutingLoadBalancer.handleNameResolutionError( - Status.UNKNOWN.withDescription("unknown error")); - assertThat(balancer.upstreamError).isNull(); - } - - private FakeLoadBalancer deliverAddressesAndUpdateToRemoveChildPolicy( - Route route, String childPolicyName) { - lbConfigInventory.put("actionX", null); - Route routeX = - new Route( - new RouteMatch( - new PathMatcher( - "/XService/xMethod", null, null), - Collections.emptyList(), - null), - "actionX"); - deliverResolvedAddresses( - ImmutableMap.of(route, childPolicyName, routeX, "policy_x")); - - verify(helper, atLeastOnce()).updateBalancingState( - eq(ConnectivityState.CONNECTING), any(SubchannelPicker.class)); - assertThat(childBalancers).hasSize(2); - FakeLoadBalancer balancer = childBalancers.get(0); - - deliverResolvedAddresses(ImmutableMap.of(routeX, "policy_x")); - verify(helper, atLeast(2)).updateBalancingState( - eq(ConnectivityState.CONNECTING), any(SubchannelPicker.class)); - assertThat(Iterables.getOnlyElement(fakeClock.getPendingTasks()).getDelay(TimeUnit.MINUTES)) - .isEqualTo(XdsRoutingLoadBalancer.DELAYED_ACTION_DELETION_TIME_MINUTES); - return balancer; - } - - private void deliverResolvedAddresses(final Map childPolicies) { - syncContext.execute(new Runnable() { - @Override - public void run() { - xdsRoutingLoadBalancer - .handleResolvedAddresses( - ResolvedAddresses.newBuilder() - .setAddresses(Collections.emptyList()) - .setLoadBalancingPolicyConfig(buildConfig(childPolicies)) - .build()); - } - }); - } - - private XdsRoutingConfig buildConfig(Map childPolicies) { - Map childPolicySelections = new LinkedHashMap<>(); - List routeList = new ArrayList<>(); - for (Route route : childPolicies.keySet()) { - String childActionName = route.getActionName(); - String childPolicyName = childPolicies.get(route); - Object childConfig = lbConfigInventory.get(childActionName); - PolicySelection policy = - new PolicySelection(new FakeLoadBalancerProvider(childPolicyName), childConfig); - childPolicySelections.put(childActionName, policy); - routeList.add(route); - } - return new XdsRoutingConfig(routeList, childPolicySelections); - } - - @Test - public void routeMatchingSubchannelPicker_typicalRouting() { - Subchannel subchannel1 = mock(Subchannel.class); - Subchannel subchannel2 = mock(Subchannel.class); - Subchannel subchannel3 = mock(Subchannel.class); - RouteMatchingSubchannelPicker routeMatchingPicker = - new RouteMatchingSubchannelPicker( - ImmutableMap.of( - routeMatch1, pickerOf(subchannel1), - routeMatch2, pickerOf(subchannel2), - routeMatch3, pickerOf(subchannel3))); - - PickSubchannelArgs args1 = - createPickSubchannelArgs( - "FooService", "barMethod", - ImmutableMap.of("user-agent", "gRPC-Java", "grpc-encoding", "gzip")); - assertThat(routeMatchingPicker.pickSubchannel(args1).getSubchannel()) - .isSameInstanceAs(subchannel1); - - PickSubchannelArgs args2 = - createPickSubchannelArgs( - "FooService", "bazMethod", - ImmutableMap.of("user-agent", "gRPC-Java", "custom-key", "custom-value")); - assertThat(routeMatchingPicker.pickSubchannel(args2).getSubchannel()) - .isSameInstanceAs(subchannel2); - - PickSubchannelArgs args3 = - createPickSubchannelArgs( - "FooService", "barMethod", - ImmutableMap.of("user-agent", "gRPC-Java", "custom-key", "custom-value")); - assertThat(routeMatchingPicker.pickSubchannel(args3).getSubchannel()) - .isSameInstanceAs(subchannel3); - - PickSubchannelArgs args4 = - createPickSubchannelArgs( - "BazService", "fooMethod", - Collections.emptyMap()); - assertThat(routeMatchingPicker.pickSubchannel(args4).getSubchannel()) - .isSameInstanceAs(subchannel3); - } - - private static SubchannelPicker pickerOf(final Subchannel subchannel) { - return new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withSubchannel(subchannel); - } - }; - } - - private static PickSubchannelArgs createPickSubchannelArgs( - String service, String method, Map headers) { - MethodDescriptor methodDescriptor = - MethodDescriptor.newBuilder() - .setType(MethodType.UNARY).setFullMethodName(service + "/" + method) - .setRequestMarshaller(TestMethodDescriptors.voidMarshaller()) - .setResponseMarshaller(TestMethodDescriptors.voidMarshaller()) - .build(); - Metadata metadata = new Metadata(); - for (Map.Entry entry : headers.entrySet()) { - metadata.put( - Metadata.Key.of(entry.getKey(), Metadata.ASCII_STRING_MARSHALLER), entry.getValue()); - } - return new PickSubchannelArgsImpl(methodDescriptor, metadata, CallOptions.DEFAULT); - } - - private final class FakeLoadBalancerProvider extends LoadBalancerProvider { - private final String policyName; - - FakeLoadBalancerProvider(String policyName) { - this.policyName = policyName; - } - - @Override - public LoadBalancer newLoadBalancer(Helper helper) { - FakeLoadBalancer balancer = new FakeLoadBalancer(policyName, helper); - childBalancers.add(balancer); - return balancer; - } - - @Override - public boolean isAvailable() { - return true; - } - - @Override - public int getPriority() { - return 0; // doesn't matter - } - - @Override - public String getPolicyName() { - return policyName; - } - } - - private final class FakeLoadBalancer extends LoadBalancer { - private final String name; - private final Helper helper; - private Object config; - private Status upstreamError; - private boolean shutdown; - - FakeLoadBalancer(String name, Helper helper) { - this.name = name; - this.helper = helper; - } - - @Override - public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { - config = resolvedAddresses.getLoadBalancingPolicyConfig(); - } - - @Override - public void handleNameResolutionError(Status error) { - upstreamError = error; - } - - @Override - public void shutdown() { - shutdown = true; - childBalancers.remove(this); - } - - void deliverSubchannelState(final Subchannel subchannel, ConnectivityState state) { - SubchannelPicker picker = new SubchannelPicker() { - @Override - public PickResult pickSubchannel(PickSubchannelArgs args) { - return PickResult.withSubchannel(subchannel); - } - }; - helper.updateBalancingState(state, picker); - } - } -} From 2e411512bea11b037bb8011625bf5a371a05f521 Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Thu, 24 Sep 2020 10:48:58 -0700 Subject: [PATCH 66/86] xds: rename CDS/EDS resource watch interface (#7454) --- .../java/io/grpc/xds/CdsLoadBalancer.java | 12 +- .../java/io/grpc/xds/EdsLoadBalancer2.java | 12 +- xds/src/main/java/io/grpc/xds/XdsClient.java | 66 +-- .../main/java/io/grpc/xds/XdsClientImpl.java | 224 ++++---- .../java/io/grpc/xds/CdsLoadBalancerTest.java | 18 +- .../io/grpc/xds/EdsLoadBalancer2Test.java | 16 +- .../java/io/grpc/xds/XdsClientImplTest.java | 512 +++++++++--------- .../java/io/grpc/xds/XdsClientImplTestV2.java | 512 +++++++++--------- 8 files changed, 676 insertions(+), 696 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java index 8515a83214b..8ad0b0de9ad 100644 --- a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer.java @@ -35,8 +35,8 @@ import io.grpc.xds.CdsLoadBalancerProvider.CdsConfig; import io.grpc.xds.EdsLoadBalancerProvider.EdsConfig; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; -import io.grpc.xds.XdsClient.ClusterUpdate; -import io.grpc.xds.XdsClient.ClusterWatcher; +import io.grpc.xds.XdsClient.CdsResourceWatcher; +import io.grpc.xds.XdsClient.CdsUpdate; import io.grpc.xds.XdsLogger.XdsLogLevel; import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import io.grpc.xds.internal.sds.SslContextProviderSupplier; @@ -177,19 +177,19 @@ protected Helper delegate() { } } - private final class CdsLbState implements ClusterWatcher { + private final class CdsLbState implements CdsResourceWatcher { private final ChannelSecurityLbHelper lbHelper = new ChannelSecurityLbHelper(); @Nullable LoadBalancer edsBalancer; private CdsLbState() { - xdsClient.watchClusterData(clusterName, this); + xdsClient.watchCdsResource(clusterName, this); logger.log(XdsLogLevel.INFO, "Started watcher for cluster {0} with xDS client {1}", clusterName, xdsClient); } @Override - public void onClusterChanged(ClusterUpdate newUpdate) { + public void onChanged(CdsUpdate newUpdate) { if (logger.isLoggable(XdsLogLevel.INFO)) { logger.log( XdsLogLevel.INFO, @@ -270,7 +270,7 @@ public void onError(Status error) { } void shutdown() { - xdsClient.cancelClusterDataWatch(clusterName, this); + xdsClient.cancelCdsResourceWatch(clusterName, this); logger.log(XdsLogLevel.INFO, "Cancelled watcher for cluster {0} with xDS client {1}", clusterName, xdsClient); if (edsBalancer != null) { diff --git a/xds/src/main/java/io/grpc/xds/EdsLoadBalancer2.java b/xds/src/main/java/io/grpc/xds/EdsLoadBalancer2.java index a02578530fc..ece7baf742c 100644 --- a/xds/src/main/java/io/grpc/xds/EdsLoadBalancer2.java +++ b/xds/src/main/java/io/grpc/xds/EdsLoadBalancer2.java @@ -45,8 +45,8 @@ import io.grpc.xds.ThreadSafeRandom.ThreadSafeRandomImpl; import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedPolicySelection; import io.grpc.xds.WeightedTargetLoadBalancerProvider.WeightedTargetConfig; -import io.grpc.xds.XdsClient.EndpointUpdate; -import io.grpc.xds.XdsClient.EndpointWatcher; +import io.grpc.xds.XdsClient.EdsResourceWatcher; +import io.grpc.xds.XdsClient.EdsUpdate; import io.grpc.xds.XdsLogger.XdsLogLevel; import io.grpc.xds.XdsSubchannelPickers.ErrorPicker; import java.util.ArrayList; @@ -146,7 +146,7 @@ public LoadBalancer newLoadBalancer(Helper helper) { return new ChildLbState(helper); } - private final class ChildLbState extends LoadBalancer implements EndpointWatcher { + private final class ChildLbState extends LoadBalancer implements EdsResourceWatcher { @Nullable private final LoadStatsStore loadStatsStore; private final DropHandlingLbHelper lbHelper; @@ -170,7 +170,7 @@ private ChildLbState(Helper helper) { logger.log( XdsLogLevel.INFO, "Start endpoint watcher on {0} with xDS client {1}", resourceName, xdsClient); - xdsClient.watchEndpointData(resourceName, this); + xdsClient.watchEdsResource(resourceName, this); } @Override @@ -221,7 +221,7 @@ public void shutdown() { xdsClient.cancelClientStatsReport(); xdsClient.removeClientStats(cluster, edsServiceName); } - xdsClient.cancelEndpointDataWatch(resourceName, this); + xdsClient.cancelEdsResourceWatch(resourceName, this); logger.log( XdsLogLevel.INFO, "Cancelled endpoint watcher on {0} with xDS client {1}", resourceName, xdsClient); @@ -236,7 +236,7 @@ public boolean canHandleEmptyAddressListFromNameResolution() { } @Override - public void onEndpointChanged(EndpointUpdate update) { + public void onChanged(EdsUpdate update) { logger.log(XdsLogLevel.DEBUG, "Received endpoint update from xDS client {0}: {1}", xdsClient, update); if (logger.isLoggable(XdsLogLevel.INFO)) { diff --git a/xds/src/main/java/io/grpc/xds/XdsClient.java b/xds/src/main/java/io/grpc/xds/XdsClient.java index 606a47f9068..7ec7afdc8b4 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClient.java +++ b/xds/src/main/java/io/grpc/xds/XdsClient.java @@ -206,12 +206,7 @@ public String toString() { } } - /** - * Data class containing the results of performing a resource discovery RPC via CDS protocol. - * The results include configurations for a single upstream cluster, such as endpoint discovery - * type, load balancing policy, connection timeout and etc. - */ - static final class ClusterUpdate { + static final class CdsUpdate { private final String clusterName; @Nullable private final String edsServiceName; @@ -220,7 +215,7 @@ static final class ClusterUpdate { private final String lrsServerName; private final UpstreamTlsContext upstreamTlsContext; - private ClusterUpdate( + private CdsUpdate( String clusterName, @Nullable String edsServiceName, String lbPolicy, @@ -295,7 +290,7 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { return false; } - ClusterUpdate that = (ClusterUpdate) o; + CdsUpdate that = (CdsUpdate) o; return Objects.equals(clusterName, that.clusterName) && Objects.equals(edsServiceName, that.edsServiceName) && Objects.equals(lbPolicy, that.lbPolicy) @@ -317,7 +312,6 @@ static final class Builder { @Nullable private UpstreamTlsContext upstreamTlsContext; - // Use ClusterUpdate.newBuilder(). private Builder() { } @@ -346,29 +340,23 @@ Builder setUpstreamTlsContext(UpstreamTlsContext upstreamTlsContext) { return this; } - ClusterUpdate build() { + CdsUpdate build() { checkState(clusterName != null, "clusterName is not set"); checkState(lbPolicy != null, "lbPolicy is not set"); return - new ClusterUpdate( + new CdsUpdate( clusterName, edsServiceName, lbPolicy, lrsServerName, upstreamTlsContext); } } } - /** - * Data class containing the results of performing a resource discovery RPC via EDS protocol. - * The results include endpoint addresses running the requested service, as well as - * configurations for traffic control such as drop overloads, inter-cluster load balancing - * policy and etc. - */ - static final class EndpointUpdate { + static final class EdsUpdate { private final String clusterName; private final Map localityLbEndpointsMap; private final List dropPolicies; - private EndpointUpdate( + private EdsUpdate( String clusterName, Map localityLbEndpoints, List dropPolicies) { @@ -407,7 +395,7 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { return false; } - EndpointUpdate that = (EndpointUpdate) o; + EdsUpdate that = (EdsUpdate) o; return Objects.equals(clusterName, that.clusterName) && Objects.equals(localityLbEndpointsMap, that.localityLbEndpointsMap) && Objects.equals(dropPolicies, that.dropPolicies); @@ -434,7 +422,6 @@ static final class Builder { private Map localityLbEndpointsMap = new LinkedHashMap<>(); private List dropPolicies = new ArrayList<>(); - // Use EndpointUpdate.newBuilder(). private Builder() { } @@ -453,10 +440,10 @@ Builder addDropPolicy(DropOverload policy) { return this; } - EndpointUpdate build() { + EdsUpdate build() { checkState(clusterName != null, "clusterName is not set"); return - new EndpointUpdate( + new EdsUpdate( clusterName, ImmutableMap.copyOf(localityLbEndpointsMap), ImmutableList.copyOf(dropPolicies)); @@ -539,6 +526,7 @@ interface RdsResourceWatcher extends ResourceWatcher { /** * Config watcher interface. To be implemented by the xDS resolver. */ + // TODO(chengyuanzhang): delete me. interface ConfigWatcher extends ResourceWatcher { /** @@ -547,20 +535,14 @@ interface ConfigWatcher extends ResourceWatcher { void onConfigChanged(ConfigUpdate update); } - /** - * Cluster watcher interface. - */ - interface ClusterWatcher extends ResourceWatcher { + interface CdsResourceWatcher extends ResourceWatcher { - void onClusterChanged(ClusterUpdate update); + void onChanged(CdsUpdate update); } - /** - * Endpoint watcher interface. - */ - interface EndpointWatcher extends ResourceWatcher { + interface EdsResourceWatcher extends ResourceWatcher { - void onEndpointChanged(EndpointUpdate update); + void onChanged(EdsUpdate update); } /** @@ -619,29 +601,27 @@ void cancelRdsResourceWatch(String resourceName, RdsResourceWatcher watcher) { } /** - * Registers a data watcher for the given cluster. + * Registers a data watcher for the given CDS resource. */ - void watchClusterData(String clusterName, ClusterWatcher watcher) { + void watchCdsResource(String resourceName, CdsResourceWatcher watcher) { } /** - * Unregisters the given cluster watcher, which was registered to receive updates for the - * given cluster. + * Unregisters the given CDS resource watcher. */ - void cancelClusterDataWatch(String clusterName, ClusterWatcher watcher) { + void cancelCdsResourceWatch(String resourceName, CdsResourceWatcher watcher) { } /** - * Registers a data watcher for endpoints in the given cluster. + * Registers a data watcher for the given EDS resource. */ - void watchEndpointData(String clusterName, EndpointWatcher watcher) { + void watchEdsResource(String resourceName, EdsResourceWatcher watcher) { } /** - * Unregisters the given endpoints watcher, which was registered to receive updates for - * endpoints information in the given cluster. + * Unregisters the given EDS resource watcher. */ - void cancelEndpointDataWatch(String clusterName, EndpointWatcher watcher) { + void cancelEdsResourceWatch(String resourceName, EdsResourceWatcher watcher) { } /** diff --git a/xds/src/main/java/io/grpc/xds/XdsClientImpl.java b/xds/src/main/java/io/grpc/xds/XdsClientImpl.java index ba43da1796d..b01b01acc79 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientImpl.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientImpl.java @@ -127,29 +127,29 @@ final class XdsClientImpl extends XdsClient { private Node node; // Cached data for CDS responses, keyed by cluster names. - // Optimization: cache ClusterUpdate, which contains only information needed by gRPC, instead + // Optimization: cache CdsUpdate, which contains only information needed by gRPC, instead // of whole Cluster messages to reduce memory usage. - private final Map clusterNamesToClusterUpdates = new HashMap<>(); + private final Map clusterNamesToCdsUpdates = new HashMap<>(); // Cached CDS resources that are known to be absent. private final Set absentCdsResources = new HashSet<>(); // Cached data for EDS responses, keyed by cluster names. // CDS responses indicate absence of clusters and EDS responses indicate presence of clusters. - // Optimization: cache EndpointUpdate, which contains only information needed by gRPC, instead + // Optimization: cache EdsUpdate, which contains only information needed by gRPC, instead // of whole ClusterLoadAssignment messages to reduce memory usage. - private final Map clusterNamesToEndpointUpdates = new HashMap<>(); + private final Map clusterNamesToEdsUpdates = new HashMap<>(); // Cached EDS resources that are known to be absent. private final Set absentEdsResources = new HashSet<>(); // Cluster watchers waiting for cluster information updates. Multiple cluster watchers // can watch on information for the same cluster. - private final Map> clusterWatchers = new HashMap<>(); + private final Map> cdsWatchers = new HashMap<>(); // Endpoint watchers waiting for endpoint updates for each cluster. Multiple endpoint // watchers can watch endpoints in the same cluster. - private final Map> endpointWatchers = new HashMap<>(); + private final Map> edsWatchers = new HashMap<>(); // Resource fetch timers are used to conclude absence of resources. Each timer is activated when // subscription for the resource starts and disarmed on first update for the resource. @@ -281,27 +281,27 @@ void watchConfigData(String targetAuthority, ConfigWatcher watcher) { } @Override - void watchClusterData(String clusterName, ClusterWatcher watcher) { - checkNotNull(clusterName, "clusterName"); + void watchCdsResource(String resourceName, CdsResourceWatcher watcher) { + checkNotNull(resourceName, "resourceName"); checkNotNull(watcher, "watcher"); boolean needRequest = false; - if (!clusterWatchers.containsKey(clusterName)) { - logger.log(XdsLogLevel.INFO, "Start watching cluster {0}", clusterName); + if (!cdsWatchers.containsKey(resourceName)) { + logger.log(XdsLogLevel.INFO, "Start watching cluster {0}", resourceName); needRequest = true; - clusterWatchers.put(clusterName, new HashSet()); + cdsWatchers.put(resourceName, new HashSet()); } - Set watchers = clusterWatchers.get(clusterName); - checkState(!watchers.contains(watcher), "watcher for %s already registered", clusterName); + Set watchers = cdsWatchers.get(resourceName); + checkState(!watchers.contains(watcher), "watcher for %s already registered", resourceName); watchers.add(watcher); // If local cache contains cluster information to be watched, notify the watcher immediately. - if (absentCdsResources.contains(clusterName)) { - logger.log(XdsLogLevel.DEBUG, "Cluster resource {0} is known to be absent", clusterName); - watcher.onResourceDoesNotExist(clusterName); + if (absentCdsResources.contains(resourceName)) { + logger.log(XdsLogLevel.DEBUG, "Cluster resource {0} is known to be absent", resourceName); + watcher.onResourceDoesNotExist(resourceName); return; } - if (clusterNamesToClusterUpdates.containsKey(clusterName)) { - logger.log(XdsLogLevel.DEBUG, "Retrieve cluster info {0} from local cache", clusterName); - watcher.onClusterChanged(clusterNamesToClusterUpdates.get(clusterName)); + if (clusterNamesToCdsUpdates.containsKey(resourceName)) { + logger.log(XdsLogLevel.DEBUG, "Retrieve cluster info {0} from local cache", resourceName); + watcher.onChanged(clusterNamesToCdsUpdates.get(resourceName)); return; } @@ -313,34 +313,34 @@ void watchClusterData(String clusterName, ClusterWatcher watcher) { if (adsStream == null) { startRpcStream(); } - adsStream.sendXdsRequest(ResourceType.CDS, clusterWatchers.keySet()); + adsStream.sendXdsRequest(ResourceType.CDS, cdsWatchers.keySet()); ScheduledHandle timeoutHandle = syncContext .schedule( - new CdsResourceFetchTimeoutTask(clusterName), + new CdsResourceFetchTimeoutTask(resourceName), INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS, timeService); - cdsRespTimers.put(clusterName, timeoutHandle); + cdsRespTimers.put(resourceName, timeoutHandle); } } @Override - void cancelClusterDataWatch(String clusterName, ClusterWatcher watcher) { + void cancelCdsResourceWatch(String resourceName, CdsResourceWatcher watcher) { checkNotNull(watcher, "watcher"); - Set watchers = clusterWatchers.get(clusterName); + Set watchers = cdsWatchers.get(resourceName); checkState( watchers != null && watchers.contains(watcher), - "watcher for %s was not registered", clusterName); + "watcher for %s was not registered", resourceName); watchers.remove(watcher); if (watchers.isEmpty()) { - logger.log(XdsLogLevel.INFO, "Stop watching cluster {0}", clusterName); - clusterWatchers.remove(clusterName); + logger.log(XdsLogLevel.INFO, "Stop watching cluster {0}", resourceName); + cdsWatchers.remove(resourceName); // Remove the corresponding CDS entry. - absentCdsResources.remove(clusterName); - clusterNamesToClusterUpdates.remove(clusterName); + absentCdsResources.remove(resourceName); + clusterNamesToCdsUpdates.remove(resourceName); // Cancel and delete response timer waiting for the corresponding resource. - if (cdsRespTimers.containsKey(clusterName)) { - cdsRespTimers.get(clusterName).cancel(); - cdsRespTimers.remove(clusterName); + if (cdsRespTimers.containsKey(resourceName)) { + cdsRespTimers.get(resourceName).cancel(); + cdsRespTimers.remove(resourceName); } // No longer interested in this cluster, send an updated CDS request to unsubscribe // this resource. @@ -350,36 +350,36 @@ void cancelClusterDataWatch(String clusterName, ClusterWatcher watcher) { } checkState(adsStream != null, "Severe bug: ADS stream was not created while an endpoint watcher was registered"); - adsStream.sendXdsRequest(ResourceType.CDS, clusterWatchers.keySet()); + adsStream.sendXdsRequest(ResourceType.CDS, cdsWatchers.keySet()); } } @Override - void watchEndpointData(String clusterName, EndpointWatcher watcher) { + void watchEdsResource(String resourceName, EdsResourceWatcher watcher) { checkNotNull(watcher, "watcher"); boolean needRequest = false; - if (!endpointWatchers.containsKey(clusterName)) { - logger.log(XdsLogLevel.INFO, "Start watching endpoints in cluster {0}", clusterName); + if (!edsWatchers.containsKey(resourceName)) { + logger.log(XdsLogLevel.INFO, "Start watching endpoints in cluster {0}", resourceName); needRequest = true; - endpointWatchers.put(clusterName, new HashSet()); + edsWatchers.put(resourceName, new HashSet()); } - Set watchers = endpointWatchers.get(clusterName); - checkState(!watchers.contains(watcher), "watcher for %s already registered", clusterName); + Set watchers = edsWatchers.get(resourceName); + checkState(!watchers.contains(watcher), "watcher for %s already registered", resourceName); watchers.add(watcher); // If local cache contains endpoint information for the cluster to be watched, notify // the watcher immediately. - if (absentEdsResources.contains(clusterName)) { + if (absentEdsResources.contains(resourceName)) { logger.log( XdsLogLevel.DEBUG, - "Endpoint resource for cluster {0} is known to be absent.", clusterName); - watcher.onResourceDoesNotExist(clusterName); + "Endpoint resource for cluster {0} is known to be absent.", resourceName); + watcher.onResourceDoesNotExist(resourceName); return; } - if (clusterNamesToEndpointUpdates.containsKey(clusterName)) { + if (clusterNamesToEdsUpdates.containsKey(resourceName)) { logger.log( XdsLogLevel.DEBUG, - "Retrieve endpoints info for cluster {0} from local cache.", clusterName); - watcher.onEndpointChanged(clusterNamesToEndpointUpdates.get(clusterName)); + "Retrieve endpoints info for cluster {0} from local cache.", resourceName); + watcher.onChanged(clusterNamesToEdsUpdates.get(resourceName)); return; } @@ -391,34 +391,34 @@ void watchEndpointData(String clusterName, EndpointWatcher watcher) { if (adsStream == null) { startRpcStream(); } - adsStream.sendXdsRequest(ResourceType.EDS, endpointWatchers.keySet()); + adsStream.sendXdsRequest(ResourceType.EDS, edsWatchers.keySet()); ScheduledHandle timeoutHandle = syncContext .schedule( - new EdsResourceFetchTimeoutTask(clusterName), + new EdsResourceFetchTimeoutTask(resourceName), INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS, timeService); - edsRespTimers.put(clusterName, timeoutHandle); + edsRespTimers.put(resourceName, timeoutHandle); } } @Override - void cancelEndpointDataWatch(String clusterName, EndpointWatcher watcher) { + void cancelEdsResourceWatch(String resourceName, EdsResourceWatcher watcher) { checkNotNull(watcher, "watcher"); - Set watchers = endpointWatchers.get(clusterName); + Set watchers = edsWatchers.get(resourceName); checkState( watchers != null && watchers.contains(watcher), - "watcher for %s was not registered", clusterName); + "watcher for %s was not registered", resourceName); watchers.remove(watcher); if (watchers.isEmpty()) { - logger.log(XdsLogLevel.INFO, "Stop watching endpoints in cluster {0}", clusterName); - endpointWatchers.remove(clusterName); + logger.log(XdsLogLevel.INFO, "Stop watching endpoints in cluster {0}", resourceName); + edsWatchers.remove(resourceName); // Remove the corresponding EDS cache entry. - absentEdsResources.remove(clusterName); - clusterNamesToEndpointUpdates.remove(clusterName); + absentEdsResources.remove(resourceName); + clusterNamesToEdsUpdates.remove(resourceName); // Cancel and delete response timer waiting for the corresponding resource. - if (edsRespTimers.containsKey(clusterName)) { - edsRespTimers.get(clusterName).cancel(); - edsRespTimers.remove(clusterName); + if (edsRespTimers.containsKey(resourceName)) { + edsRespTimers.get(resourceName).cancel(); + edsRespTimers.remove(resourceName); } // No longer interested in this cluster, send an updated EDS request to unsubscribe // this resource. @@ -426,7 +426,7 @@ void cancelEndpointDataWatch(String clusterName, EndpointWatcher watcher) { // Currently in retry backoff. return; } - adsStream.sendXdsRequest(ResourceType.EDS, endpointWatchers.keySet()); + adsStream.sendXdsRequest(ResourceType.EDS, edsWatchers.keySet()); } } @@ -949,7 +949,7 @@ private void handleCdsResponse(DiscoveryResponseData cdsResponse) { } catch (InvalidProtocolBufferException e) { logger.log(XdsLogLevel.WARNING, "Failed to unpack Clusters in CDS response {0}", e); adsStream.sendNackRequest( - ResourceType.CDS, clusterWatchers.keySet(), + ResourceType.CDS, cdsWatchers.keySet(), cdsResponse.getVersionInfo(), "Malformed CDS response: " + e); return; } @@ -957,7 +957,7 @@ private void handleCdsResponse(DiscoveryResponseData cdsResponse) { String errorMessage = null; // Cluster information update for requested clusters received in this CDS response. - Map clusterUpdates = new HashMap<>(); + Map cdsUpdates = new HashMap<>(); // CDS responses represents the state of the world, EDS services not referenced by // Clusters are those no longer exist. Set edsServices = new HashSet<>(); @@ -967,10 +967,10 @@ private void handleCdsResponse(DiscoveryResponseData cdsResponse) { // Management server is required to always send newly requested resources, even if they // may have been sent previously (proactively). Thus, client does not need to cache // unrequested resources. - if (!clusterWatchers.containsKey(clusterName)) { + if (!cdsWatchers.containsKey(clusterName)) { continue; } - ClusterUpdate.Builder updateBuilder = ClusterUpdate.newBuilder(); + CdsUpdate.Builder updateBuilder = CdsUpdate.newBuilder(); updateBuilder.setClusterName(clusterName); // The type field must be set to EDS. if (!cluster.getType().equals(DiscoveryType.EDS)) { @@ -1020,48 +1020,48 @@ private void handleCdsResponse(DiscoveryResponseData cdsResponse) { errorMessage = "Cluster " + clusterName + " : " + e.getMessage(); break; } - clusterUpdates.put(clusterName, updateBuilder.build()); + cdsUpdates.put(clusterName, updateBuilder.build()); } if (errorMessage != null) { adsStream.sendNackRequest( ResourceType.CDS, - clusterWatchers.keySet(), + cdsWatchers.keySet(), cdsResponse.getVersionInfo(), errorMessage); return; } - adsStream.sendAckRequest(ResourceType.CDS, clusterWatchers.keySet(), + adsStream.sendAckRequest(ResourceType.CDS, cdsWatchers.keySet(), cdsResponse.getVersionInfo()); // Update local CDS cache with data in this response. - absentCdsResources.removeAll(clusterUpdates.keySet()); - for (Map.Entry entry : clusterNamesToClusterUpdates.entrySet()) { - if (!clusterUpdates.containsKey(entry.getKey())) { + absentCdsResources.removeAll(cdsUpdates.keySet()); + for (Map.Entry entry : clusterNamesToCdsUpdates.entrySet()) { + if (!cdsUpdates.containsKey(entry.getKey())) { // Some previously existing resource no longer exists. absentCdsResources.add(entry.getKey()); - } else if (clusterUpdates.get(entry.getKey()).equals(entry.getValue())) { - clusterUpdates.remove(entry.getKey()); + } else if (cdsUpdates.get(entry.getKey()).equals(entry.getValue())) { + cdsUpdates.remove(entry.getKey()); } } - clusterNamesToClusterUpdates.keySet().removeAll(absentCdsResources); - clusterNamesToClusterUpdates.putAll(clusterUpdates); + clusterNamesToCdsUpdates.keySet().removeAll(absentCdsResources); + clusterNamesToCdsUpdates.putAll(cdsUpdates); // Remove EDS cache entries for ClusterLoadAssignments not referenced by this CDS response. - for (String clusterName : clusterNamesToEndpointUpdates.keySet()) { + for (String clusterName : clusterNamesToEdsUpdates.keySet()) { if (!edsServices.contains(clusterName)) { absentEdsResources.add(clusterName); // Notify EDS resource removal to watchers. - if (endpointWatchers.containsKey(clusterName)) { - Set watchers = endpointWatchers.get(clusterName); - for (EndpointWatcher watcher : watchers) { + if (edsWatchers.containsKey(clusterName)) { + Set watchers = edsWatchers.get(clusterName); + for (EdsResourceWatcher watcher : watchers) { watcher.onResourceDoesNotExist(clusterName); } } } } - clusterNamesToEndpointUpdates.keySet().retainAll(edsServices); + clusterNamesToEdsUpdates.keySet().retainAll(edsServices); - for (String clusterName : clusterUpdates.keySet()) { + for (String clusterName : cdsUpdates.keySet()) { if (cdsRespTimers.containsKey(clusterName)) { cdsRespTimers.get(clusterName).cancel(); cdsRespTimers.remove(clusterName); @@ -1069,17 +1069,17 @@ private void handleCdsResponse(DiscoveryResponseData cdsResponse) { } // Notify watchers if clusters interested in present in this CDS response. - for (Map.Entry> entry : clusterWatchers.entrySet()) { + for (Map.Entry> entry : cdsWatchers.entrySet()) { String clusterName = entry.getKey(); - if (clusterUpdates.containsKey(entry.getKey())) { - ClusterUpdate clusterUpdate = clusterUpdates.get(clusterName); - for (ClusterWatcher watcher : entry.getValue()) { - watcher.onClusterChanged(clusterUpdate); + if (cdsUpdates.containsKey(entry.getKey())) { + CdsUpdate cdsUpdate = cdsUpdates.get(clusterName); + for (CdsResourceWatcher watcher : entry.getValue()) { + watcher.onChanged(cdsUpdate); } - } else if (!clusterNamesToClusterUpdates.containsKey(entry.getKey()) + } else if (!clusterNamesToCdsUpdates.containsKey(entry.getKey()) && !cdsRespTimers.containsKey(clusterName)) { // Update for previously present resource being removed. - for (ClusterWatcher watcher : entry.getValue()) { + for (CdsResourceWatcher watcher : entry.getValue()) { watcher.onResourceDoesNotExist(entry.getKey()); } } @@ -1124,7 +1124,7 @@ private void handleEdsResponse(DiscoveryResponseData edsResponse) { logger.log( XdsLogLevel.WARNING, "Failed to unpack ClusterLoadAssignments in EDS response {0}", e); adsStream.sendNackRequest( - ResourceType.EDS, endpointWatchers.keySet(), + ResourceType.EDS, edsWatchers.keySet(), edsResponse.getVersionInfo(), "Malformed EDS response: " + e); return; } @@ -1132,7 +1132,7 @@ private void handleEdsResponse(DiscoveryResponseData edsResponse) { String errorMessage = null; // Endpoint information updates for requested clusters received in this EDS response. - Map endpointUpdates = new HashMap<>(); + Map edsUpdates = new HashMap<>(); // Walk through each ClusterLoadAssignment message. If any of them for requested clusters // contain invalid information for gRPC's load balancing usage, the whole response is rejected. for (ClusterLoadAssignment assignment : clusterLoadAssignments) { @@ -1141,10 +1141,10 @@ private void handleEdsResponse(DiscoveryResponseData edsResponse) { // Management server is required to always send newly requested resources, even if they // may have been sent previously (proactively). Thus, client does not need to cache // unrequested resources. - if (!endpointWatchers.containsKey(clusterName)) { + if (!edsWatchers.containsKey(clusterName)) { continue; } - EndpointUpdate.Builder updateBuilder = EndpointUpdate.newBuilder(); + EdsUpdate.Builder updateBuilder = EdsUpdate.newBuilder(); updateBuilder.setClusterName(clusterName); Set priorities = new HashSet<>(); int maxPriority = -1; @@ -1192,37 +1192,37 @@ private void handleEdsResponse(DiscoveryResponseData edsResponse) { : assignment.getPolicy().getDropOverloadsList()) { updateBuilder.addDropPolicy(DropOverload.fromEnvoyProtoDropOverload(dropOverload)); } - EndpointUpdate update = updateBuilder.build(); - endpointUpdates.put(clusterName, update); + EdsUpdate update = updateBuilder.build(); + edsUpdates.put(clusterName, update); } if (errorMessage != null) { adsStream.sendNackRequest( ResourceType.EDS, - endpointWatchers.keySet(), + edsWatchers.keySet(), edsResponse.getVersionInfo(), errorMessage); return; } - adsStream.sendAckRequest(ResourceType.EDS, endpointWatchers.keySet(), + adsStream.sendAckRequest(ResourceType.EDS, edsWatchers.keySet(), edsResponse.getVersionInfo()); // Update local EDS cache by inserting updated endpoint information. - clusterNamesToEndpointUpdates.putAll(endpointUpdates); - absentEdsResources.removeAll(endpointUpdates.keySet()); + clusterNamesToEdsUpdates.putAll(edsUpdates); + absentEdsResources.removeAll(edsUpdates.keySet()); // Notify watchers waiting for updates of endpoint information received in this EDS response. // Based on xDS protocol, the management server should not send endpoint data again if // nothing has changed. - for (Map.Entry entry : endpointUpdates.entrySet()) { + for (Map.Entry entry : edsUpdates.entrySet()) { String clusterName = entry.getKey(); // Cancel and delete response timeout timer. if (edsRespTimers.containsKey(clusterName)) { edsRespTimers.get(clusterName).cancel(); edsRespTimers.remove(clusterName); } - if (endpointWatchers.containsKey(clusterName)) { - for (EndpointWatcher watcher : endpointWatchers.get(clusterName)) { - watcher.onEndpointChanged(entry.getValue()); + if (edsWatchers.containsKey(clusterName)) { + for (EdsResourceWatcher watcher : edsWatchers.get(clusterName)) { + watcher.onChanged(entry.getValue()); } } } @@ -1249,9 +1249,9 @@ public void run() { new ListenerResourceFetchTimeoutTask(":" + listenerPort), INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS, timeService); } - if (!clusterWatchers.isEmpty()) { - adsStream.sendXdsRequest(ResourceType.CDS, clusterWatchers.keySet()); - for (String clusterName : clusterWatchers.keySet()) { + if (!cdsWatchers.isEmpty()) { + adsStream.sendXdsRequest(ResourceType.CDS, cdsWatchers.keySet()); + for (String clusterName : cdsWatchers.keySet()) { ScheduledHandle timeoutHandle = syncContext .schedule( @@ -1260,9 +1260,9 @@ public void run() { cdsRespTimers.put(clusterName, timeoutHandle); } } - if (!endpointWatchers.isEmpty()) { - adsStream.sendXdsRequest(ResourceType.EDS, endpointWatchers.keySet()); - for (String clusterName : endpointWatchers.keySet()) { + if (!edsWatchers.isEmpty()) { + adsStream.sendXdsRequest(ResourceType.EDS, edsWatchers.keySet()); + for (String clusterName : edsWatchers.keySet()) { ScheduledHandle timeoutHandle = syncContext .schedule( @@ -1521,13 +1521,13 @@ private void handleStreamClosed(Status error) { if (listenerWatcher != null) { listenerWatcher.onError(error); } - for (Set watchers : clusterWatchers.values()) { - for (ClusterWatcher watcher : watchers) { + for (Set watchers : cdsWatchers.values()) { + for (CdsResourceWatcher watcher : watchers) { watcher.onError(error); } } - for (Set watchers : endpointWatchers.values()) { - for (EndpointWatcher watcher : watchers) { + for (Set watchers : edsWatchers.values()) { + for (EdsResourceWatcher watcher : watchers) { watcher.onError(error); } } @@ -1910,7 +1910,7 @@ public void run() { super.run(); cdsRespTimers.remove(resourceName); absentCdsResources.add(resourceName); - for (ClusterWatcher wat : clusterWatchers.get(resourceName)) { + for (CdsResourceWatcher wat : cdsWatchers.get(resourceName)) { wat.onResourceDoesNotExist(resourceName); } } @@ -1928,7 +1928,7 @@ public void run() { super.run(); edsRespTimers.remove(resourceName); absentEdsResources.add(resourceName); - for (EndpointWatcher wat : endpointWatchers.get(resourceName)) { + for (EdsResourceWatcher wat : edsWatchers.get(resourceName)) { wat.onResourceDoesNotExist(resourceName); } } diff --git a/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java index d7570725e0b..831ff06d5c3 100644 --- a/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/CdsLoadBalancerTest.java @@ -306,17 +306,17 @@ private static List createEndpointAddresses(int n) { } private final class FakeXdsClient extends XdsClient { - private ClusterWatcher watcher; + private CdsResourceWatcher watcher; @Override - void watchClusterData(String clusterName, ClusterWatcher watcher) { - assertThat(clusterName).isEqualTo(CLUSTER); + void watchCdsResource(String resourceName, CdsResourceWatcher watcher) { + assertThat(resourceName).isEqualTo(CLUSTER); this.watcher = watcher; } @Override - void cancelClusterDataWatch(String clusterName, ClusterWatcher watcher) { - assertThat(clusterName).isEqualTo(CLUSTER); + void cancelCdsResourceWatch(String resourceName, CdsResourceWatcher watcher) { + assertThat(resourceName).isEqualTo(CLUSTER); assertThat(watcher).isSameInstanceAs(this.watcher); this.watcher = null; } @@ -331,8 +331,8 @@ void deliverClusterInfo( syncContext.execute(new Runnable() { @Override public void run() { - watcher.onClusterChanged( - ClusterUpdate.newBuilder() + watcher.onChanged( + CdsUpdate.newBuilder() .setClusterName(CLUSTER) .setEdsServiceName(edsServiceName) .setLbPolicy("round_robin") // only supported policy @@ -348,8 +348,8 @@ void deliverClusterInfo( syncContext.execute(new Runnable() { @Override public void run() { - watcher.onClusterChanged( - ClusterUpdate.newBuilder() + watcher.onChanged( + CdsUpdate.newBuilder() .setClusterName(CLUSTER) .setEdsServiceName(edsServiceName) .setLbPolicy("round_robin") // only supported policy diff --git a/xds/src/test/java/io/grpc/xds/EdsLoadBalancer2Test.java b/xds/src/test/java/io/grpc/xds/EdsLoadBalancer2Test.java index b964c06157b..ecb83c94607 100644 --- a/xds/src/test/java/io/grpc/xds/EdsLoadBalancer2Test.java +++ b/xds/src/test/java/io/grpc/xds/EdsLoadBalancer2Test.java @@ -711,7 +711,7 @@ public String toString() { } private final class FakeXdsClient extends XdsClient { - private final Map watchers = new HashMap<>(); + private final Map watchers = new HashMap<>(); private final Map> dropStats = new HashMap<>(); @Override @@ -720,13 +720,13 @@ void shutdown() { } @Override - void watchEndpointData(String clusterName, EndpointWatcher watcher) { - watchers.put(clusterName, watcher); + void watchEdsResource(String resourceName, EdsResourceWatcher watcher) { + watchers.put(resourceName, watcher); } @Override - void cancelEndpointDataWatch(String clusterName, EndpointWatcher watcher) { - watchers.remove(clusterName); + void cancelEdsResourceWatch(String resourceName, EdsResourceWatcher watcher) { + watchers.remove(resourceName); } @Override @@ -755,14 +755,14 @@ void deliverClusterLoadAssignment( @Override public void run() { if (watchers.containsKey(resource)) { - EndpointUpdate.Builder builder = EndpointUpdate.newBuilder().setClusterName(resource); + EdsUpdate.Builder builder = EdsUpdate.newBuilder().setClusterName(resource); for (DropOverload dropOverload : dropOverloads) { builder.addDropPolicy(dropOverload); } for (Locality locality : localityLbEndpointsMap.keySet()) { builder.addLocalityLbEndpoints(locality, localityLbEndpointsMap.get(locality)); } - watchers.get(resource).onEndpointChanged(builder.build()); + watchers.get(resource).onChanged(builder.build()); } } }); @@ -783,7 +783,7 @@ void deliverError(final Status error) { syncContext.execute(new Runnable() { @Override public void run() { - for (EndpointWatcher watcher : watchers.values()) { + for (EdsResourceWatcher watcher : watchers.values()) { watcher.onError(error); } } diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java index f75b67fccd2..346c7230621 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java @@ -91,12 +91,12 @@ import io.grpc.xds.EnvoyProtoData.Locality; import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints; import io.grpc.xds.EnvoyProtoData.Node; -import io.grpc.xds.XdsClient.ClusterUpdate; -import io.grpc.xds.XdsClient.ClusterWatcher; +import io.grpc.xds.XdsClient.CdsResourceWatcher; +import io.grpc.xds.XdsClient.CdsUpdate; import io.grpc.xds.XdsClient.ConfigUpdate; import io.grpc.xds.XdsClient.ConfigWatcher; -import io.grpc.xds.XdsClient.EndpointUpdate; -import io.grpc.xds.XdsClient.EndpointWatcher; +import io.grpc.xds.XdsClient.EdsResourceWatcher; +import io.grpc.xds.XdsClient.EdsUpdate; import io.grpc.xds.XdsClient.XdsChannel; import io.grpc.xds.XdsClientImpl.MessagePrinter; import java.io.IOException; @@ -206,9 +206,9 @@ public void uncaughtException(Thread t, Throwable e) { @Mock private ConfigWatcher configWatcher; @Mock - private ClusterWatcher clusterWatcher; + private CdsResourceWatcher cdsResourceWatcher; @Mock - private EndpointWatcher endpointWatcher; + private EdsResourceWatcher edsResourceWatcher; private ManagedChannel channel; private XdsClientImpl xdsClient; @@ -1337,7 +1337,7 @@ public void updateRdsRequestResourceWhileInitialResourceFetchInProgress() { */ @Test public void cdsResponseWithoutMatchingResource() { - xdsClient.watchClusterData("cluster-foo.googleapis.com", clusterWatcher); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", cdsResourceWatcher); StreamObserver responseObserver = responseObservers.poll(); StreamObserver requestObserver = requestObservers.poll(); @@ -1359,12 +1359,12 @@ public void cdsResponseWithoutMatchingResource() { verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", "cluster-foo.googleapis.com", XdsClientImpl.ADS_TYPE_URL_CDS, "0000"))); - verify(clusterWatcher, never()).onClusterChanged(any(ClusterUpdate.class)); - verify(clusterWatcher, never()).onResourceDoesNotExist("cluster-foo.googleapis.com"); - verify(clusterWatcher, never()).onError(any(Status.class)); + verify(cdsResourceWatcher, never()).onChanged(any(CdsUpdate.class)); + verify(cdsResourceWatcher, never()).onResourceDoesNotExist("cluster-foo.googleapis.com"); + verify(cdsResourceWatcher, never()).onError(any(Status.class)); fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); - verify(clusterWatcher).onResourceDoesNotExist("cluster-foo.googleapis.com"); + verify(cdsResourceWatcher).onResourceDoesNotExist("cluster-foo.googleapis.com"); assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); } @@ -1374,7 +1374,7 @@ public void cdsResponseWithoutMatchingResource() { */ @Test public void cdsResponseWithMatchingResource() { - xdsClient.watchClusterData("cluster-foo.googleapis.com", clusterWatcher); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", cdsResourceWatcher); StreamObserver responseObserver = responseObservers.poll(); StreamObserver requestObserver = requestObservers.poll(); @@ -1401,13 +1401,13 @@ public void cdsResponseWithMatchingResource() { XdsClientImpl.ADS_TYPE_URL_CDS, "0000"))); assertThat(cdsRespTimer.isCancelled()).isTrue(); - ArgumentCaptor clusterUpdateCaptor = ArgumentCaptor.forClass(null); - verify(clusterWatcher).onClusterChanged(clusterUpdateCaptor.capture()); - ClusterUpdate clusterUpdate = clusterUpdateCaptor.getValue(); - assertThat(clusterUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(clusterUpdate.getEdsServiceName()).isNull(); - assertThat(clusterUpdate.getLbPolicy()).isEqualTo("round_robin"); - assertThat(clusterUpdate.getLrsServerName()).isNull(); + ArgumentCaptor cdsUpdateCaptor = ArgumentCaptor.forClass(null); + verify(cdsResourceWatcher).onChanged(cdsUpdateCaptor.capture()); + CdsUpdate cdsUpdate = cdsUpdateCaptor.getValue(); + assertThat(cdsUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(cdsUpdate.getEdsServiceName()).isNull(); + assertThat(cdsUpdate.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate.getLrsServerName()).isNull(); // Management server sends back another CDS response updating the requested Cluster. clusters = ImmutableList.of( @@ -1424,13 +1424,13 @@ public void cdsResponseWithMatchingResource() { .onNext(eq(buildDiscoveryRequest(NODE, "1", "cluster-foo.googleapis.com", XdsClientImpl.ADS_TYPE_URL_CDS, "0001"))); - verify(clusterWatcher, times(2)).onClusterChanged(clusterUpdateCaptor.capture()); - clusterUpdate = clusterUpdateCaptor.getValue(); - assertThat(clusterUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(clusterUpdate.getEdsServiceName()) + verify(cdsResourceWatcher, times(2)).onChanged(cdsUpdateCaptor.capture()); + cdsUpdate = cdsUpdateCaptor.getValue(); + assertThat(cdsUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(cdsUpdate.getEdsServiceName()) .isEqualTo("eds-cluster-foo.googleapis.com"); - assertThat(clusterUpdate.getLbPolicy()).isEqualTo("round_robin"); - assertThat(clusterUpdate.getLrsServerName()).isEqualTo(""); + assertThat(cdsUpdate.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate.getLrsServerName()).isEqualTo(""); } /** @@ -1438,7 +1438,7 @@ public void cdsResponseWithMatchingResource() { */ @Test public void cdsResponseWithUpstreamTlsContext() { - xdsClient.watchClusterData("cluster-foo.googleapis.com", clusterWatcher); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", cdsResourceWatcher); StreamObserver responseObserver = responseObservers.poll(); StreamObserver requestObserver = requestObservers.poll(); @@ -1458,10 +1458,10 @@ public void cdsResponseWithUpstreamTlsContext() { verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "0", "cluster-foo.googleapis.com", XdsClientImpl.ADS_TYPE_URL_CDS, "0000"))); - ArgumentCaptor clusterUpdateCaptor = ArgumentCaptor.forClass(null); - verify(clusterWatcher, times(1)).onClusterChanged(clusterUpdateCaptor.capture()); - ClusterUpdate clusterUpdate = clusterUpdateCaptor.getValue(); - EnvoyServerProtoData.UpstreamTlsContext upstreamTlsContext = clusterUpdate + ArgumentCaptor cdsUpdateCaptor = ArgumentCaptor.forClass(null); + verify(cdsResourceWatcher, times(1)).onChanged(cdsUpdateCaptor.capture()); + CdsUpdate cdsUpdate = cdsUpdateCaptor.getValue(); + EnvoyServerProtoData.UpstreamTlsContext upstreamTlsContext = cdsUpdate .getUpstreamTlsContext(); SdsSecretConfig validationContextSdsSecretConfig = upstreamTlsContext.getCommonTlsContext() .getValidationContextSdsSecretConfig(); @@ -1478,13 +1478,13 @@ public void cdsResponseWithUpstreamTlsContext() { } @Test - public void multipleClusterWatchers() { - ClusterWatcher watcher1 = mock(ClusterWatcher.class); - ClusterWatcher watcher2 = mock(ClusterWatcher.class); - ClusterWatcher watcher3 = mock(ClusterWatcher.class); - xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher1); - xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher2); - xdsClient.watchClusterData("cluster-bar.googleapis.com", watcher3); + public void multipleCdsWatchers() { + CdsResourceWatcher watcher1 = mock(CdsResourceWatcher.class); + CdsResourceWatcher watcher2 = mock(CdsResourceWatcher.class); + CdsResourceWatcher watcher3 = mock(CdsResourceWatcher.class); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher1); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher2); + xdsClient.watchCdsResource("cluster-bar.googleapis.com", watcher3); StreamObserver responseObserver = responseObservers.poll(); StreamObserver requestObserver = requestObservers.poll(); @@ -1516,25 +1516,25 @@ public void multipleClusterWatchers() { XdsClientImpl.ADS_TYPE_URL_CDS, "0000"))); // Two watchers get notification of cluster update for the cluster they are interested in. - ArgumentCaptor clusterUpdateCaptor1 = ArgumentCaptor.forClass(null); - verify(watcher1).onClusterChanged(clusterUpdateCaptor1.capture()); - ClusterUpdate clusterUpdate1 = clusterUpdateCaptor1.getValue(); - assertThat(clusterUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(clusterUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(clusterUpdate1.getEdsServiceName()).isNull(); - assertThat(clusterUpdate1.getLbPolicy()).isEqualTo("round_robin"); - assertThat(clusterUpdate1.getLrsServerName()).isNull(); - - ArgumentCaptor clusterUpdateCaptor2 = ArgumentCaptor.forClass(null); - verify(watcher2).onClusterChanged(clusterUpdateCaptor2.capture()); - ClusterUpdate clusterUpdate2 = clusterUpdateCaptor2.getValue(); - assertThat(clusterUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(clusterUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(clusterUpdate2.getEdsServiceName()).isNull(); - assertThat(clusterUpdate2.getLbPolicy()).isEqualTo("round_robin"); - assertThat(clusterUpdate2.getLrsServerName()).isNull(); - - verify(watcher3, never()).onClusterChanged(any(ClusterUpdate.class)); + ArgumentCaptor cdsUpdateCaptor1 = ArgumentCaptor.forClass(null); + verify(watcher1).onChanged(cdsUpdateCaptor1.capture()); + CdsUpdate cdsUpdate1 = cdsUpdateCaptor1.getValue(); + assertThat(cdsUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(cdsUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(cdsUpdate1.getEdsServiceName()).isNull(); + assertThat(cdsUpdate1.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate1.getLrsServerName()).isNull(); + + ArgumentCaptor cdsUpdateCaptor2 = ArgumentCaptor.forClass(null); + verify(watcher2).onChanged(cdsUpdateCaptor2.capture()); + CdsUpdate cdsUpdate2 = cdsUpdateCaptor2.getValue(); + assertThat(cdsUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(cdsUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(cdsUpdate2.getEdsServiceName()).isNull(); + assertThat(cdsUpdate2.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate2.getLrsServerName()).isNull(); + + verify(watcher3, never()).onChanged(any(CdsUpdate.class)); verify(watcher3, never()).onResourceDoesNotExist("cluster-bar.googleapis.com"); verify(watcher3, never()).onError(any(Status.class)); @@ -1563,14 +1563,14 @@ public void multipleClusterWatchers() { XdsClientImpl.ADS_TYPE_URL_CDS, "0001"))); verifyNoMoreInteractions(watcher1, watcher2); // resource has no change - ArgumentCaptor clusterUpdateCaptor3 = ArgumentCaptor.forClass(null); - verify(watcher3).onClusterChanged(clusterUpdateCaptor3.capture()); - ClusterUpdate clusterUpdate3 = clusterUpdateCaptor3.getValue(); - assertThat(clusterUpdate3.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); - assertThat(clusterUpdate3.getEdsServiceName()) + ArgumentCaptor cdsUpdateCaptor3 = ArgumentCaptor.forClass(null); + verify(watcher3).onChanged(cdsUpdateCaptor3.capture()); + CdsUpdate cdsUpdate3 = cdsUpdateCaptor3.getValue(); + assertThat(cdsUpdate3.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); + assertThat(cdsUpdate3.getEdsServiceName()) .isEqualTo("eds-cluster-bar.googleapis.com"); - assertThat(clusterUpdate3.getLbPolicy()).isEqualTo("round_robin"); - assertThat(clusterUpdate3.getLrsServerName()).isEqualTo(""); + assertThat(cdsUpdate3.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate3.getLrsServerName()).isEqualTo(""); } /** @@ -1580,8 +1580,8 @@ public void multipleClusterWatchers() { */ @Test public void watchClusterAlreadyBeingWatched() { - ClusterWatcher watcher1 = mock(ClusterWatcher.class); - xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher1); + CdsResourceWatcher watcher1 = mock(CdsResourceWatcher.class); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher1); // Streaming RPC starts after a first watcher is added. StreamObserver responseObserver = responseObservers.poll(); @@ -1606,28 +1606,28 @@ public void watchClusterAlreadyBeingWatched() { .onNext(eq(buildDiscoveryRequest(NODE, "0", "cluster-foo.googleapis.com", XdsClientImpl.ADS_TYPE_URL_CDS, "0000"))); - ArgumentCaptor clusterUpdateCaptor1 = ArgumentCaptor.forClass(null); - verify(watcher1).onClusterChanged(clusterUpdateCaptor1.capture()); - ClusterUpdate clusterUpdate1 = clusterUpdateCaptor1.getValue(); - assertThat(clusterUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(clusterUpdate1.getEdsServiceName()).isNull(); - assertThat(clusterUpdate1.getLbPolicy()).isEqualTo("round_robin"); - assertThat(clusterUpdate1.getLrsServerName()).isNull(); + ArgumentCaptor cdsUpdateCaptor1 = ArgumentCaptor.forClass(null); + verify(watcher1).onChanged(cdsUpdateCaptor1.capture()); + CdsUpdate cdsUpdate1 = cdsUpdateCaptor1.getValue(); + assertThat(cdsUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(cdsUpdate1.getEdsServiceName()).isNull(); + assertThat(cdsUpdate1.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate1.getLrsServerName()).isNull(); assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); // Another cluster watcher interested in the same cluster is added. - ClusterWatcher watcher2 = mock(ClusterWatcher.class); - xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher2); + CdsResourceWatcher watcher2 = mock(CdsResourceWatcher.class); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher2); // Since the client has received cluster update for this cluster before, cached result is // notified to the newly added watcher immediately. - ArgumentCaptor clusterUpdateCaptor2 = ArgumentCaptor.forClass(null); - verify(watcher2).onClusterChanged(clusterUpdateCaptor2.capture()); - ClusterUpdate clusterUpdate2 = clusterUpdateCaptor2.getValue(); - assertThat(clusterUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(clusterUpdate2.getEdsServiceName()).isNull(); - assertThat(clusterUpdate2.getLbPolicy()).isEqualTo("round_robin"); - assertThat(clusterUpdate2.getLrsServerName()).isNull(); + ArgumentCaptor cdsUpdateCaptor2 = ArgumentCaptor.forClass(null); + verify(watcher2).onChanged(cdsUpdateCaptor2.capture()); + CdsUpdate cdsUpdate2 = cdsUpdateCaptor2.getValue(); + assertThat(cdsUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(cdsUpdate2.getEdsServiceName()).isNull(); + assertThat(cdsUpdate2.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate2.getLrsServerName()).isNull(); verifyNoMoreInteractions(requestObserver); assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); @@ -1637,9 +1637,9 @@ public void watchClusterAlreadyBeingWatched() { * Basic operations of adding/canceling cluster data watchers. */ @Test - public void addRemoveClusterWatchers() { - ClusterWatcher watcher1 = mock(ClusterWatcher.class); - xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher1); + public void addRemoveCdsWatchers() { + CdsResourceWatcher watcher1 = mock(CdsResourceWatcher.class); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher1); // Streaming RPC starts after a first watcher is added. StreamObserver responseObserver = responseObservers.poll(); @@ -1663,17 +1663,17 @@ public void addRemoveClusterWatchers() { .onNext(eq(buildDiscoveryRequest(NODE, "0", "cluster-foo.googleapis.com", XdsClientImpl.ADS_TYPE_URL_CDS, "0000"))); - ArgumentCaptor clusterUpdateCaptor1 = ArgumentCaptor.forClass(null); - verify(watcher1).onClusterChanged(clusterUpdateCaptor1.capture()); - ClusterUpdate clusterUpdate1 = clusterUpdateCaptor1.getValue(); - assertThat(clusterUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(clusterUpdate1.getEdsServiceName()).isNull(); - assertThat(clusterUpdate1.getLbPolicy()).isEqualTo("round_robin"); - assertThat(clusterUpdate1.getLrsServerName()).isNull(); + ArgumentCaptor cdsUpdateCaptor1 = ArgumentCaptor.forClass(null); + verify(watcher1).onChanged(cdsUpdateCaptor1.capture()); + CdsUpdate cdsUpdate1 = cdsUpdateCaptor1.getValue(); + assertThat(cdsUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(cdsUpdate1.getEdsServiceName()).isNull(); + assertThat(cdsUpdate1.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate1.getLrsServerName()).isNull(); // Add another cluster watcher for a different cluster. - ClusterWatcher watcher2 = mock(ClusterWatcher.class); - xdsClient.watchClusterData("cluster-bar.googleapis.com", watcher2); + CdsResourceWatcher watcher2 = mock(CdsResourceWatcher.class); + xdsClient.watchCdsResource("cluster-bar.googleapis.com", watcher2); // Client sent a new CDS request for all interested resources. verify(requestObserver) @@ -1701,17 +1701,17 @@ public void addRemoveClusterWatchers() { ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), XdsClientImpl.ADS_TYPE_URL_CDS, "0001"))); verifyNoMoreInteractions(watcher1); // resource has no change - ArgumentCaptor clusterUpdateCaptor2 = ArgumentCaptor.forClass(null); - verify(watcher2).onClusterChanged(clusterUpdateCaptor2.capture()); - ClusterUpdate clusterUpdate2 = clusterUpdateCaptor2.getValue(); - assertThat(clusterUpdate2.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); - assertThat(clusterUpdate2.getEdsServiceName()) + ArgumentCaptor cdsUpdateCaptor2 = ArgumentCaptor.forClass(null); + verify(watcher2).onChanged(cdsUpdateCaptor2.capture()); + CdsUpdate cdsUpdate2 = cdsUpdateCaptor2.getValue(); + assertThat(cdsUpdate2.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); + assertThat(cdsUpdate2.getEdsServiceName()) .isEqualTo("eds-cluster-bar.googleapis.com"); - assertThat(clusterUpdate2.getLbPolicy()).isEqualTo("round_robin"); - assertThat(clusterUpdate2.getLrsServerName()).isEqualTo(""); + assertThat(cdsUpdate2.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate2.getLrsServerName()).isEqualTo(""); // Cancel one of the watcher. - xdsClient.cancelClusterDataWatch("cluster-foo.googleapis.com", watcher1); + xdsClient.cancelCdsResourceWatch("cluster-foo.googleapis.com", watcher1); // Since the cancelled watcher was the last watcher interested in that cluster (but there // is still interested resource), client sent an new CDS request to unsubscribe from @@ -1723,7 +1723,7 @@ public void addRemoveClusterWatchers() { // Management server has nothing to respond. // Cancel the other watcher. All resources have been unsubscribed. - xdsClient.cancelClusterDataWatch("cluster-bar.googleapis.com", watcher2); + xdsClient.cancelCdsResourceWatch("cluster-bar.googleapis.com", watcher2); verify(requestObserver) .onNext( @@ -1750,9 +1750,9 @@ public void addRemoveClusterWatchers() { verifyNoMoreInteractions(watcher1, watcher2); // A new cluster watcher is added to watch cluster foo again. - ClusterWatcher watcher3 = mock(ClusterWatcher.class); - xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher3); - verify(watcher3, never()).onClusterChanged(any(ClusterUpdate.class)); + CdsResourceWatcher watcher3 = mock(CdsResourceWatcher.class); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher3); + verify(watcher3, never()).onChanged(any(CdsUpdate.class)); // A CDS request is sent to indicate subscription of "cluster-foo.googleapis.com" only. verify(requestObserver) @@ -1770,13 +1770,13 @@ public void addRemoveClusterWatchers() { responseObserver.onNext(response); // Notified with cached data immediately. - ArgumentCaptor clusterUpdateCaptor3 = ArgumentCaptor.forClass(null); - verify(watcher3).onClusterChanged(clusterUpdateCaptor3.capture()); - ClusterUpdate clusterUpdate3 = clusterUpdateCaptor3.getValue(); - assertThat(clusterUpdate3.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(clusterUpdate3.getEdsServiceName()).isNull(); - assertThat(clusterUpdate3.getLbPolicy()).isEqualTo("round_robin"); - assertThat(clusterUpdate2.getLrsServerName()).isEqualTo(""); + ArgumentCaptor cdsUpdateCaptor3 = ArgumentCaptor.forClass(null); + verify(watcher3).onChanged(cdsUpdateCaptor3.capture()); + CdsUpdate cdsUpdate3 = cdsUpdateCaptor3.getValue(); + assertThat(cdsUpdate3.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(cdsUpdate3.getEdsServiceName()).isNull(); + assertThat(cdsUpdate3.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate2.getLrsServerName()).isEqualTo(""); verifyNoMoreInteractions(watcher1, watcher2); @@ -1787,9 +1787,9 @@ public void addRemoveClusterWatchers() { } @Test - public void addRemoveClusterWatcherWhileInitialResourceFetchInProgress() { - ClusterWatcher watcher1 = mock(ClusterWatcher.class); - xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher1); + public void addRemoveCdsWatcherWhileInitialResourceFetchInProgress() { + CdsResourceWatcher watcher1 = mock(CdsResourceWatcher.class); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher1); // Streaming RPC starts after a first watcher is added. StreamObserver requestObserver = requestObservers.poll(); @@ -1804,12 +1804,12 @@ public void addRemoveClusterWatcherWhileInitialResourceFetchInProgress() { fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC - 1, TimeUnit.SECONDS); - ClusterWatcher watcher2 = mock(ClusterWatcher.class); - ClusterWatcher watcher3 = mock(ClusterWatcher.class); - ClusterWatcher watcher4 = mock(ClusterWatcher.class); - xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher2); - xdsClient.watchClusterData("cluster-bar.googleapis.com", watcher3); - xdsClient.watchClusterData("cluster-bar.googleapis.com", watcher4); + CdsResourceWatcher watcher2 = mock(CdsResourceWatcher.class); + CdsResourceWatcher watcher3 = mock(CdsResourceWatcher.class); + CdsResourceWatcher watcher4 = mock(CdsResourceWatcher.class); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher2); + xdsClient.watchCdsResource("cluster-bar.googleapis.com", watcher3); + xdsClient.watchCdsResource("cluster-bar.googleapis.com", watcher4); // Client sends a new CDS request for updating the latest resource subscription. verify(requestObserver) @@ -1828,8 +1828,8 @@ public void addRemoveClusterWatcherWhileInitialResourceFetchInProgress() { verify(watcher2).onResourceDoesNotExist("cluster-foo.googleapis.com"); // The absence result is known immediately. - ClusterWatcher watcher5 = mock(ClusterWatcher.class); - xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher5); + CdsResourceWatcher watcher5 = mock(CdsResourceWatcher.class); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher5); verify(watcher5).onResourceDoesNotExist("cluster-foo.googleapis.com"); assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); @@ -1837,9 +1837,9 @@ public void addRemoveClusterWatcherWhileInitialResourceFetchInProgress() { // Cancel watchers while discovery for resource "cluster-bar.googleapis.com" is still // in progress. - xdsClient.cancelClusterDataWatch("cluster-bar.googleapis.com", watcher3); + xdsClient.cancelCdsResourceWatch("cluster-bar.googleapis.com", watcher3); assertThat(timeoutTask.isCancelled()).isFalse(); - xdsClient.cancelClusterDataWatch("cluster-bar.googleapis.com", watcher4); + xdsClient.cancelCdsResourceWatch("cluster-bar.googleapis.com", watcher4); // Client sends a CDS request for resource subscription update (Omitted). @@ -1853,7 +1853,7 @@ public void addRemoveClusterWatcherWhileInitialResourceFetchInProgress() { @Test public void cdsUpdateForClusterBeingRemoved() { - xdsClient.watchClusterData("cluster-foo.googleapis.com", clusterWatcher); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", cdsResourceWatcher); StreamObserver responseObserver = responseObservers.poll(); StreamObserver requestObserver = requestObservers.poll(); @@ -1871,13 +1871,13 @@ public void cdsUpdateForClusterBeingRemoved() { // Client sent an ACK CDS request (Omitted). - ArgumentCaptor clusterUpdateCaptor = ArgumentCaptor.forClass(null); - verify(clusterWatcher).onClusterChanged(clusterUpdateCaptor.capture()); - ClusterUpdate clusterUpdate = clusterUpdateCaptor.getValue(); - assertThat(clusterUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(clusterUpdate.getEdsServiceName()).isNull(); - assertThat(clusterUpdate.getLbPolicy()).isEqualTo("round_robin"); - assertThat(clusterUpdate.getLrsServerName()).isEqualTo(""); + ArgumentCaptor cdsUpdateCaptor = ArgumentCaptor.forClass(null); + verify(cdsResourceWatcher).onChanged(cdsUpdateCaptor.capture()); + CdsUpdate cdsUpdate = cdsUpdateCaptor.getValue(); + assertThat(cdsUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(cdsUpdate.getEdsServiceName()).isNull(); + assertThat(cdsUpdate.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate.getLrsServerName()).isEqualTo(""); assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); // No cluster is available. @@ -1886,7 +1886,7 @@ public void cdsUpdateForClusterBeingRemoved() { XdsClientImpl.ADS_TYPE_URL_CDS, "0001"); responseObserver.onNext(response); - verify(clusterWatcher).onResourceDoesNotExist("cluster-foo.googleapis.com"); + verify(cdsResourceWatcher).onResourceDoesNotExist("cluster-foo.googleapis.com"); } /** @@ -1898,7 +1898,7 @@ public void cdsUpdateForClusterBeingRemoved() { */ @Test public void edsResponseWithoutMatchingResource() { - xdsClient.watchEndpointData("cluster-foo.googleapis.com", endpointWatcher); + xdsClient.watchEdsResource("cluster-foo.googleapis.com", edsResourceWatcher); StreamObserver responseObserver = responseObservers.poll(); StreamObserver requestObserver = requestObservers.poll(); @@ -1936,11 +1936,11 @@ public void edsResponseWithoutMatchingResource() { .onNext(eq(buildDiscoveryRequest(NODE, "0", "cluster-foo.googleapis.com", XdsClientImpl.ADS_TYPE_URL_EDS, "0000"))); - verify(endpointWatcher, never()).onEndpointChanged(any(EndpointUpdate.class)); - verify(endpointWatcher, never()).onResourceDoesNotExist("cluster-foo.googleapis.com"); - verify(endpointWatcher, never()).onError(any(Status.class)); + verify(edsResourceWatcher, never()).onChanged(any(EdsUpdate.class)); + verify(edsResourceWatcher, never()).onResourceDoesNotExist("cluster-foo.googleapis.com"); + verify(edsResourceWatcher, never()).onError(any(Status.class)); fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); - verify(endpointWatcher).onResourceDoesNotExist("cluster-foo.googleapis.com"); + verify(edsResourceWatcher).onResourceDoesNotExist("cluster-foo.googleapis.com"); assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); } @@ -1950,7 +1950,7 @@ public void edsResponseWithoutMatchingResource() { */ @Test public void edsResponseWithMatchingResource() { - xdsClient.watchEndpointData("cluster-foo.googleapis.com", endpointWatcher); + xdsClient.watchEdsResource("cluster-foo.googleapis.com", edsResourceWatcher); StreamObserver responseObserver = responseObservers.poll(); StreamObserver requestObserver = requestObservers.poll(); @@ -2002,15 +2002,15 @@ public void edsResponseWithMatchingResource() { .onNext(eq(buildDiscoveryRequest(NODE, "0", "cluster-foo.googleapis.com", XdsClientImpl.ADS_TYPE_URL_EDS, "0000"))); - ArgumentCaptor endpointUpdateCaptor = ArgumentCaptor.forClass(null); - verify(endpointWatcher).onEndpointChanged(endpointUpdateCaptor.capture()); - EndpointUpdate endpointUpdate = endpointUpdateCaptor.getValue(); - assertThat(endpointUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(endpointUpdate.getDropPolicies()) + ArgumentCaptor edsUpdateCaptor = ArgumentCaptor.forClass(null); + verify(edsResourceWatcher).onChanged(edsUpdateCaptor.capture()); + EdsUpdate edsUpdate = edsUpdateCaptor.getValue(); + assertThat(edsUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(edsUpdate.getDropPolicies()) .containsExactly( new DropOverload("lb", 200), new DropOverload("throttle", 1000)); - assertThat(endpointUpdate.getLocalityLbEndpointsMap()) + assertThat(edsUpdate.getLocalityLbEndpointsMap()) .containsExactly( new Locality("region1", "zone1", "subzone1"), new LocalityLbEndpoints( @@ -2035,21 +2035,21 @@ public void edsResponseWithMatchingResource() { .onNext(eq(buildDiscoveryRequest(NODE, "1", "cluster-foo.googleapis.com", XdsClientImpl.ADS_TYPE_URL_EDS, "0001"))); - verify(endpointWatcher, times(2)).onEndpointChanged(endpointUpdateCaptor.capture()); - endpointUpdate = endpointUpdateCaptor.getValue(); - assertThat(endpointUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(endpointUpdate.getDropPolicies()).isEmpty(); - assertThat(endpointUpdate.getLocalityLbEndpointsMap()).isEmpty(); + verify(edsResourceWatcher, times(2)).onChanged(edsUpdateCaptor.capture()); + edsUpdate = edsUpdateCaptor.getValue(); + assertThat(edsUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(edsUpdate.getDropPolicies()).isEmpty(); + assertThat(edsUpdate.getLocalityLbEndpointsMap()).isEmpty(); } @Test - public void multipleEndpointWatchers() { - EndpointWatcher watcher1 = mock(EndpointWatcher.class); - EndpointWatcher watcher2 = mock(EndpointWatcher.class); - EndpointWatcher watcher3 = mock(EndpointWatcher.class); - xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher1); - xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher2); - xdsClient.watchEndpointData("cluster-bar.googleapis.com", watcher3); + public void multipleEdsWatchers() { + EdsResourceWatcher watcher1 = mock(EdsResourceWatcher.class); + EdsResourceWatcher watcher2 = mock(EdsResourceWatcher.class); + EdsResourceWatcher watcher3 = mock(EdsResourceWatcher.class); + xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher1); + xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher2); + xdsClient.watchEdsResource("cluster-bar.googleapis.com", watcher3); StreamObserver responseObserver = responseObservers.poll(); StreamObserver requestObserver = requestObservers.poll(); @@ -2091,11 +2091,11 @@ public void multipleEndpointWatchers() { XdsClientImpl.ADS_TYPE_URL_EDS, "0000"))); // Two watchers get notification of endpoint update for the cluster they are interested in. - ArgumentCaptor endpointUpdateCaptor1 = ArgumentCaptor.forClass(null); - verify(watcher1).onEndpointChanged(endpointUpdateCaptor1.capture()); - EndpointUpdate endpointUpdate1 = endpointUpdateCaptor1.getValue(); - assertThat(endpointUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(endpointUpdate1.getLocalityLbEndpointsMap()) + ArgumentCaptor edsUpdateCaptor1 = ArgumentCaptor.forClass(null); + verify(watcher1).onChanged(edsUpdateCaptor1.capture()); + EdsUpdate edsUpdate1 = edsUpdateCaptor1.getValue(); + assertThat(edsUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(edsUpdate1.getLocalityLbEndpointsMap()) .containsExactly( new Locality("region1", "zone1", "subzone1"), new LocalityLbEndpoints( @@ -2103,11 +2103,11 @@ public void multipleEndpointWatchers() { new LbEndpoint("192.168.0.1", 8080, 2, true)), 1, 0)); - ArgumentCaptor endpointUpdateCaptor2 = ArgumentCaptor.forClass(null); - verify(watcher1).onEndpointChanged(endpointUpdateCaptor2.capture()); - EndpointUpdate endpointUpdate2 = endpointUpdateCaptor2.getValue(); - assertThat(endpointUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(endpointUpdate2.getLocalityLbEndpointsMap()) + ArgumentCaptor edsUpdateCaptor2 = ArgumentCaptor.forClass(null); + verify(watcher1).onChanged(edsUpdateCaptor2.capture()); + EdsUpdate edsUpdate2 = edsUpdateCaptor2.getValue(); + assertThat(edsUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(edsUpdate2.getLocalityLbEndpointsMap()) .containsExactly( new Locality("region1", "zone1", "subzone1"), new LocalityLbEndpoints( @@ -2141,11 +2141,11 @@ public void multipleEndpointWatchers() { XdsClientImpl.ADS_TYPE_URL_EDS, "0001"))); // The corresponding watcher gets notified. - ArgumentCaptor endpointUpdateCaptor3 = ArgumentCaptor.forClass(null); - verify(watcher3).onEndpointChanged(endpointUpdateCaptor3.capture()); - EndpointUpdate endpointUpdate3 = endpointUpdateCaptor3.getValue(); - assertThat(endpointUpdate3.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); - assertThat(endpointUpdate3.getLocalityLbEndpointsMap()) + ArgumentCaptor edsUpdateCaptor3 = ArgumentCaptor.forClass(null); + verify(watcher3).onChanged(edsUpdateCaptor3.capture()); + EdsUpdate edsUpdate3 = edsUpdateCaptor3.getValue(); + assertThat(edsUpdate3.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); + assertThat(edsUpdate3.getLocalityLbEndpointsMap()) .containsExactly( new Locality("region2", "zone2", "subzone2"), new LocalityLbEndpoints( @@ -2161,8 +2161,8 @@ public void multipleEndpointWatchers() { */ @Test public void watchEndpointsForClusterAlreadyBeingWatched() { - EndpointWatcher watcher1 = mock(EndpointWatcher.class); - xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher1); + EdsResourceWatcher watcher1 = mock(EdsResourceWatcher.class); + xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher1); StreamObserver responseObserver = responseObservers.poll(); StreamObserver requestObserver = requestObservers.poll(); @@ -2196,12 +2196,12 @@ public void watchEndpointsForClusterAlreadyBeingWatched() { .onNext(eq(buildDiscoveryRequest(NODE, "0", "cluster-foo.googleapis.com", XdsClientImpl.ADS_TYPE_URL_EDS, "0000"))); - ArgumentCaptor endpointUpdateCaptor1 = ArgumentCaptor.forClass(null); - verify(watcher1).onEndpointChanged(endpointUpdateCaptor1.capture()); - EndpointUpdate endpointUpdate1 = endpointUpdateCaptor1.getValue(); - assertThat(endpointUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(endpointUpdate1.getDropPolicies()).isEmpty(); - assertThat(endpointUpdate1.getLocalityLbEndpointsMap()) + ArgumentCaptor edsUpdateCaptor1 = ArgumentCaptor.forClass(null); + verify(watcher1).onChanged(edsUpdateCaptor1.capture()); + EdsUpdate edsUpdate1 = edsUpdateCaptor1.getValue(); + assertThat(edsUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(edsUpdate1.getDropPolicies()).isEmpty(); + assertThat(edsUpdate1.getLocalityLbEndpointsMap()) .containsExactly( new Locality("region1", "zone1", "subzone1"), new LocalityLbEndpoints( @@ -2210,17 +2210,17 @@ public void watchEndpointsForClusterAlreadyBeingWatched() { 2, true)), 1, 0)); // A second endpoint watcher is registered for endpoints in the same cluster. - EndpointWatcher watcher2 = mock(EndpointWatcher.class); - xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher2); + EdsResourceWatcher watcher2 = mock(EdsResourceWatcher.class); + xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher2); // Cached endpoint information is notified to the new watcher immediately, without sending // another EDS request. - ArgumentCaptor endpointUpdateCaptor2 = ArgumentCaptor.forClass(null); - verify(watcher2).onEndpointChanged(endpointUpdateCaptor2.capture()); - EndpointUpdate endpointUpdate2 = endpointUpdateCaptor2.getValue(); - assertThat(endpointUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(endpointUpdate2.getDropPolicies()).isEmpty(); - assertThat(endpointUpdate2.getLocalityLbEndpointsMap()) + ArgumentCaptor edsUpdateCaptor2 = ArgumentCaptor.forClass(null); + verify(watcher2).onChanged(edsUpdateCaptor2.capture()); + EdsUpdate edsUpdate2 = edsUpdateCaptor2.getValue(); + assertThat(edsUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(edsUpdate2.getDropPolicies()).isEmpty(); + assertThat(edsUpdate2.getLocalityLbEndpointsMap()) .containsExactly( new Locality("region1", "zone1", "subzone1"), new LocalityLbEndpoints( @@ -2236,9 +2236,9 @@ public void watchEndpointsForClusterAlreadyBeingWatched() { * Basic operations of adding/canceling endpoint data watchers. */ @Test - public void addRemoveEndpointWatchers() { - EndpointWatcher watcher1 = mock(EndpointWatcher.class); - xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher1); + public void addRemoveEdsWatchers() { + EdsResourceWatcher watcher1 = mock(EdsResourceWatcher.class); + xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher1); // Streaming RPC starts after a first watcher is added. StreamObserver responseObserver = responseObservers.poll(); @@ -2271,11 +2271,11 @@ public void addRemoveEndpointWatchers() { .onNext(eq(buildDiscoveryRequest(NODE, "0", "cluster-foo.googleapis.com", XdsClientImpl.ADS_TYPE_URL_EDS, "0000"))); - ArgumentCaptor endpointUpdateCaptor1 = ArgumentCaptor.forClass(null); - verify(watcher1).onEndpointChanged(endpointUpdateCaptor1.capture()); - EndpointUpdate endpointUpdate1 = endpointUpdateCaptor1.getValue(); - assertThat(endpointUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(endpointUpdate1.getLocalityLbEndpointsMap()) + ArgumentCaptor edsUpdateCaptor1 = ArgumentCaptor.forClass(null); + verify(watcher1).onChanged(edsUpdateCaptor1.capture()); + EdsUpdate edsUpdate1 = edsUpdateCaptor1.getValue(); + assertThat(edsUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(edsUpdate1.getLocalityLbEndpointsMap()) .containsExactly( new Locality("region1", "zone1", "subzone1"), new LocalityLbEndpoints( @@ -2285,8 +2285,8 @@ public void addRemoveEndpointWatchers() { 1, 0)); // Add another endpoint watcher for a different cluster. - EndpointWatcher watcher2 = mock(EndpointWatcher.class); - xdsClient.watchEndpointData("cluster-bar.googleapis.com", watcher2); + EdsResourceWatcher watcher2 = mock(EdsResourceWatcher.class); + xdsClient.watchEdsResource("cluster-bar.googleapis.com", watcher2); // Client sent a new EDS request for all interested resources. verify(requestObserver) @@ -2319,11 +2319,11 @@ public void addRemoveEndpointWatchers() { ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), XdsClientImpl.ADS_TYPE_URL_EDS, "0001"))); - ArgumentCaptor endpointUpdateCaptor2 = ArgumentCaptor.forClass(null); - verify(watcher2).onEndpointChanged(endpointUpdateCaptor2.capture()); - EndpointUpdate endpointUpdate2 = endpointUpdateCaptor2.getValue(); - assertThat(endpointUpdate2.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); - assertThat(endpointUpdate2.getLocalityLbEndpointsMap()) + ArgumentCaptor edsUpdateCaptor2 = ArgumentCaptor.forClass(null); + verify(watcher2).onChanged(edsUpdateCaptor2.capture()); + EdsUpdate edsUpdate2 = edsUpdateCaptor2.getValue(); + assertThat(edsUpdate2.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); + assertThat(edsUpdate2.getLocalityLbEndpointsMap()) .containsExactly( new Locality("region2", "zone2", "subzone2"), new LocalityLbEndpoints( @@ -2332,7 +2332,7 @@ public void addRemoveEndpointWatchers() { 6, 0)); // Cancel one of the watcher. - xdsClient.cancelEndpointDataWatch("cluster-foo.googleapis.com", watcher1); + xdsClient.cancelEdsResourceWatch("cluster-foo.googleapis.com", watcher1); // Since the cancelled watcher was the last watcher interested in that cluster, client // sent an new EDS request to unsubscribe from that cluster. @@ -2343,7 +2343,7 @@ public void addRemoveEndpointWatchers() { // Management server should not respond as it had previously sent the requested resource. // Cancel the other watcher. - xdsClient.cancelEndpointDataWatch("cluster-bar.googleapis.com", watcher2); + xdsClient.cancelEdsResourceWatch("cluster-bar.googleapis.com", watcher2); // Since the cancelled watcher was the last watcher interested in that cluster, client // sent an new EDS request to unsubscribe from that cluster. @@ -2389,13 +2389,13 @@ public void addRemoveEndpointWatchers() { verifyNoMoreInteractions(watcher1, watcher2); // A new endpoint watcher is added to watch an old but was no longer interested in cluster. - EndpointWatcher watcher3 = mock(EndpointWatcher.class); - xdsClient.watchEndpointData("cluster-bar.googleapis.com", watcher3); + EdsResourceWatcher watcher3 = mock(EdsResourceWatcher.class); + xdsClient.watchEdsResource("cluster-bar.googleapis.com", watcher3); // Nothing should be notified to the new watcher as we are still waiting management server's // latest response. // Cached endpoint data should have been purged. - verify(watcher3, never()).onEndpointChanged(any(EndpointUpdate.class)); + verify(watcher3, never()).onChanged(any(EdsUpdate.class)); // An EDS request is sent to re-subscribe the cluster again. verify(requestObserver) @@ -2416,11 +2416,11 @@ public void addRemoveEndpointWatchers() { XdsClientImpl.ADS_TYPE_URL_EDS, "0003"); responseObserver.onNext(response); - ArgumentCaptor endpointUpdateCaptor3 = ArgumentCaptor.forClass(null); - verify(watcher3).onEndpointChanged(endpointUpdateCaptor3.capture()); - EndpointUpdate endpointUpdate3 = endpointUpdateCaptor3.getValue(); - assertThat(endpointUpdate3.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); - assertThat(endpointUpdate3.getLocalityLbEndpointsMap()) + ArgumentCaptor edsUpdateCaptor3 = ArgumentCaptor.forClass(null); + verify(watcher3).onChanged(edsUpdateCaptor3.capture()); + EdsUpdate edsUpdate3 = edsUpdateCaptor3.getValue(); + assertThat(edsUpdate3.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); + assertThat(edsUpdate3.getLocalityLbEndpointsMap()) .containsExactly( new Locality("region4", "zone4", "subzone4"), new LocalityLbEndpoints( @@ -2438,9 +2438,9 @@ public void addRemoveEndpointWatchers() { } @Test - public void addRemoveEndpointWatcherWhileInitialResourceFetchInProgress() { - EndpointWatcher watcher1 = mock(EndpointWatcher.class); - xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher1); + public void addRemoveEdsWatcherWhileInitialResourceFetchInProgress() { + EdsResourceWatcher watcher1 = mock(EdsResourceWatcher.class); + xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher1); // Streaming RPC starts after a first watcher is added. StreamObserver requestObserver = requestObservers.poll(); @@ -2455,12 +2455,12 @@ public void addRemoveEndpointWatcherWhileInitialResourceFetchInProgress() { fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC - 1, TimeUnit.SECONDS); - EndpointWatcher watcher2 = mock(EndpointWatcher.class); - EndpointWatcher watcher3 = mock(EndpointWatcher.class); - EndpointWatcher watcher4 = mock(EndpointWatcher.class); - xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher2); - xdsClient.watchEndpointData("cluster-bar.googleapis.com", watcher3); - xdsClient.watchEndpointData("cluster-bar.googleapis.com", watcher4); + EdsResourceWatcher watcher2 = mock(EdsResourceWatcher.class); + EdsResourceWatcher watcher3 = mock(EdsResourceWatcher.class); + EdsResourceWatcher watcher4 = mock(EdsResourceWatcher.class); + xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher2); + xdsClient.watchEdsResource("cluster-bar.googleapis.com", watcher3); + xdsClient.watchEdsResource("cluster-bar.googleapis.com", watcher4); // Client sends a new EDS request for updating the latest resource subscription. verify(requestObserver) @@ -2479,8 +2479,8 @@ public void addRemoveEndpointWatcherWhileInitialResourceFetchInProgress() { verify(watcher2).onResourceDoesNotExist("cluster-foo.googleapis.com"); // The absence result is known immediately. - EndpointWatcher watcher5 = mock(EndpointWatcher.class); - xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher5); + EdsResourceWatcher watcher5 = mock(EdsResourceWatcher.class); + xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher5); verify(watcher5).onResourceDoesNotExist("cluster-foo.googleapis.com"); assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); @@ -2488,9 +2488,9 @@ public void addRemoveEndpointWatcherWhileInitialResourceFetchInProgress() { // Cancel watchers while discovery for resource "cluster-bar.googleapis.com" is still // in progress. - xdsClient.cancelEndpointDataWatch("cluster-bar.googleapis.com", watcher3); + xdsClient.cancelEdsResourceWatch("cluster-bar.googleapis.com", watcher3); assertThat(timeoutTask.isCancelled()).isFalse(); - xdsClient.cancelEndpointDataWatch("cluster-bar.googleapis.com", watcher4); + xdsClient.cancelEdsResourceWatch("cluster-bar.googleapis.com", watcher4); // Client sends an EDS request for resource subscription update (Omitted). @@ -2504,7 +2504,7 @@ public void addRemoveEndpointWatcherWhileInitialResourceFetchInProgress() { @Test public void cdsUpdateForEdsServiceNameChange() { - xdsClient.watchClusterData("cluster-foo.googleapis.com", clusterWatcher); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", cdsResourceWatcher); StreamObserver responseObserver = responseObservers.poll(); // Management server sends back a CDS response containing requested resource. @@ -2514,7 +2514,7 @@ public void cdsUpdateForEdsServiceNameChange() { buildDiscoveryResponse("0", clusters, XdsClientImpl.ADS_TYPE_URL_CDS, "0000"); responseObserver.onNext(response); - xdsClient.watchEndpointData("cluster-foo:service-bar", endpointWatcher); + xdsClient.watchEdsResource("cluster-foo:service-bar", edsResourceWatcher); // Management server sends back an EDS response for resource "cluster-foo:service-bar". List clusterLoadAssignments = ImmutableList.of( @@ -2530,12 +2530,12 @@ public void cdsUpdateForEdsServiceNameChange() { XdsClientImpl.ADS_TYPE_URL_EDS, "0000"); responseObserver.onNext(response); - ArgumentCaptor endpointUpdateCaptor = ArgumentCaptor.forClass(null); - verify(endpointWatcher).onEndpointChanged(endpointUpdateCaptor.capture()); - EndpointUpdate endpointUpdate = endpointUpdateCaptor.getValue(); - assertThat(endpointUpdate.getClusterName()).isEqualTo("cluster-foo:service-bar"); - assertThat(endpointUpdate.getDropPolicies()).isEmpty(); - assertThat(endpointUpdate.getLocalityLbEndpointsMap()) + ArgumentCaptor edsUpdateCaptor = ArgumentCaptor.forClass(null); + verify(edsResourceWatcher).onChanged(edsUpdateCaptor.capture()); + EdsUpdate edsUpdate = edsUpdateCaptor.getValue(); + assertThat(edsUpdate.getClusterName()).isEqualTo("cluster-foo:service-bar"); + assertThat(edsUpdate.getDropPolicies()).isEmpty(); + assertThat(edsUpdate.getLocalityLbEndpointsMap()) .containsExactly( new Locality("region1", "zone1", "subzone1"), new LocalityLbEndpoints( @@ -2552,7 +2552,7 @@ public void cdsUpdateForEdsServiceNameChange() { responseObserver.onNext(response); // Watcher get notification for endpoint resource "cluster-foo:service-bar" being deleted. - verify(endpointWatcher).onResourceDoesNotExist("cluster-foo:service-bar"); + verify(edsResourceWatcher).onResourceDoesNotExist("cluster-foo:service-bar"); } /** @@ -2737,7 +2737,7 @@ public void streamClosedAndRetry() { ArgumentCaptor statusCaptor = ArgumentCaptor.forClass(null); // Start watching cluster information. - xdsClient.watchClusterData("cluster.googleapis.com", clusterWatcher); + xdsClient.watchCdsResource("cluster.googleapis.com", cdsResourceWatcher); // Client sent first CDS request. verify(requestObserver) @@ -2745,7 +2745,7 @@ public void streamClosedAndRetry() { XdsClientImpl.ADS_TYPE_URL_CDS, ""))); // Start watching endpoint information. - xdsClient.watchEndpointData("cluster.googleapis.com", endpointWatcher); + xdsClient.watchEdsResource("cluster.googleapis.com", edsResourceWatcher); // Client sent first EDS request. verify(requestObserver) @@ -2756,9 +2756,9 @@ public void streamClosedAndRetry() { responseObserver.onError(Status.UNKNOWN.asException()); verify(configWatcher).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNKNOWN); - verify(clusterWatcher).onError(statusCaptor.capture()); + verify(cdsResourceWatcher).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNKNOWN); - verify(endpointWatcher).onError(statusCaptor.capture()); + verify(edsResourceWatcher).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNKNOWN); // Resets backoff and retry immediately. @@ -2784,9 +2784,9 @@ public void streamClosedAndRetry() { responseObserver.onError(Status.UNAVAILABLE.asException()); verify(configWatcher, times(2)).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); - verify(clusterWatcher, times(2)).onError(statusCaptor.capture()); + verify(cdsResourceWatcher, times(2)).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); - verify(endpointWatcher, times(2)).onError(statusCaptor.capture()); + verify(edsResourceWatcher, times(2)).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); inOrder.verify(backoffPolicy1).nextBackoffNanos(); assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); @@ -2813,9 +2813,9 @@ public void streamClosedAndRetry() { responseObserver.onError(Status.UNAVAILABLE.asException()); verify(configWatcher, times(3)).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); - verify(clusterWatcher, times(3)).onError(statusCaptor.capture()); + verify(cdsResourceWatcher, times(3)).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); - verify(endpointWatcher, times(3)).onError(statusCaptor.capture()); + verify(edsResourceWatcher, times(3)).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); inOrder.verify(backoffPolicy1).nextBackoffNanos(); assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); @@ -2850,8 +2850,8 @@ public void streamClosedAndRetry() { // Management server closes the RPC stream. responseObserver.onCompleted(); verify(configWatcher, times(4)).onError(any(Status.class)); - verify(clusterWatcher, times(4)).onError(any(Status.class)); - verify(endpointWatcher, times(4)).onError(any(Status.class)); + verify(cdsResourceWatcher, times(4)).onError(any(Status.class)); + verify(edsResourceWatcher, times(4)).onError(any(Status.class)); // Resets backoff and retry immediately inOrder.verify(backoffPolicyProvider).get(); @@ -2875,9 +2875,9 @@ public void streamClosedAndRetry() { responseObserver.onError(Status.UNAVAILABLE.asException()); verify(configWatcher, times(5)).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); - verify(clusterWatcher, times(5)).onError(statusCaptor.capture()); + verify(cdsResourceWatcher, times(5)).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); - verify(endpointWatcher, times(5)).onError(statusCaptor.capture()); + verify(edsResourceWatcher, times(5)).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); inOrder.verify(backoffPolicy2).nextBackoffNanos(); assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); @@ -2944,7 +2944,7 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); // Start watching cluster information while RPC stream is still in retry backoff. - xdsClient.watchClusterData("cluster.googleapis.com", clusterWatcher); + xdsClient.watchCdsResource("cluster.googleapis.com", cdsResourceWatcher); // Retry after backoff. fakeClock.forwardNanos(9L); @@ -2968,7 +2968,7 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); // Start watching endpoint information while RPC stream is still in retry backoff. - xdsClient.watchEndpointData("cluster.googleapis.com", endpointWatcher); + xdsClient.watchEdsResource("cluster.googleapis.com", edsResourceWatcher); // Retry after backoff. fakeClock.forwardNanos(99L); @@ -2999,14 +2999,14 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { // Client sent an CDS ACK request (Omitted). // No longer interested in endpoint information after RPC resumes. - xdsClient.cancelEndpointDataWatch("cluster.googleapis.com", endpointWatcher); + xdsClient.cancelEdsResourceWatch("cluster.googleapis.com", edsResourceWatcher); // Client updates EDS resource subscription immediately. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", ImmutableList.of(), XdsClientImpl.ADS_TYPE_URL_EDS, ""))); // Become interested in endpoints of another cluster. - xdsClient.watchEndpointData("cluster2.googleapis.com", endpointWatcher); + xdsClient.watchEdsResource("cluster2.googleapis.com", edsResourceWatcher); // Client updates EDS resource subscription immediately. verify(requestObserver) .onNext(eq(buildDiscoveryRequest(NODE, "", "cluster2.googleapis.com", @@ -3038,8 +3038,8 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); // No longer interested in previous cluster and endpoints in that cluster. - xdsClient.cancelClusterDataWatch("cluster.googleapis.com", clusterWatcher); - xdsClient.cancelEndpointDataWatch("cluster2.googleapis.com", endpointWatcher); + xdsClient.cancelCdsResourceWatch("cluster.googleapis.com", cdsResourceWatcher); + xdsClient.cancelEdsResourceWatch("cluster2.googleapis.com", edsResourceWatcher); // Retry after backoff. fakeClock.forwardNanos(19L); @@ -3162,7 +3162,7 @@ public void streamClosedAndRetryReschedulesAllResourceFetchTimer() { // Client/server resumed LDS/RDS request/response (Omitted). // Start watching cluster data. - xdsClient.watchClusterData("cluster-foo.googleapis.com", clusterWatcher); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", cdsResourceWatcher); ScheduledTask cdsRespTimeoutTask = Iterables.getOnlyElement( fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); @@ -3190,7 +3190,7 @@ public void streamClosedAndRetryReschedulesAllResourceFetchTimer() { .isEqualTo(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC); // Start watching endpoint data. - xdsClient.watchEndpointData("cluster-foo.googleapis.com", endpointWatcher); + xdsClient.watchEdsResource("cluster-foo.googleapis.com", edsResourceWatcher); ScheduledTask edsTimeoutTask = Iterables.getOnlyElement( fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java index 17351d00d17..e04cd739aba 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java @@ -91,12 +91,12 @@ import io.grpc.xds.EnvoyProtoData.Locality; import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints; import io.grpc.xds.EnvoyProtoData.Node; -import io.grpc.xds.XdsClient.ClusterUpdate; -import io.grpc.xds.XdsClient.ClusterWatcher; +import io.grpc.xds.XdsClient.CdsResourceWatcher; +import io.grpc.xds.XdsClient.CdsUpdate; import io.grpc.xds.XdsClient.ConfigUpdate; import io.grpc.xds.XdsClient.ConfigWatcher; -import io.grpc.xds.XdsClient.EndpointUpdate; -import io.grpc.xds.XdsClient.EndpointWatcher; +import io.grpc.xds.XdsClient.EdsResourceWatcher; +import io.grpc.xds.XdsClient.EdsUpdate; import io.grpc.xds.XdsClient.XdsChannel; import io.grpc.xds.XdsClientImpl.MessagePrinter; import java.io.IOException; @@ -205,9 +205,9 @@ public void uncaughtException(Thread t, Throwable e) { @Mock private ConfigWatcher configWatcher; @Mock - private ClusterWatcher clusterWatcher; + private CdsResourceWatcher cdsResourceWatcher; @Mock - private EndpointWatcher endpointWatcher; + private EdsResourceWatcher edsResourceWatcher; private ManagedChannel channel; private XdsClientImpl xdsClient; @@ -1347,7 +1347,7 @@ public void updateRdsRequestResourceWhileInitialResourceFetchInProgress() { */ @Test public void cdsResponseWithoutMatchingResource() { - xdsClient.watchClusterData("cluster-foo.googleapis.com", clusterWatcher); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", cdsResourceWatcher); StreamObserver responseObserver = responseObservers.poll(); StreamObserver requestObserver = requestObservers.poll(); @@ -1369,12 +1369,12 @@ public void cdsResponseWithoutMatchingResource() { verify(requestObserver) .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); - verify(clusterWatcher, never()).onClusterChanged(any(ClusterUpdate.class)); - verify(clusterWatcher, never()).onResourceDoesNotExist("cluster-foo.googleapis.com"); - verify(clusterWatcher, never()).onError(any(Status.class)); + verify(cdsResourceWatcher, never()).onChanged(any(CdsUpdate.class)); + verify(cdsResourceWatcher, never()).onResourceDoesNotExist("cluster-foo.googleapis.com"); + verify(cdsResourceWatcher, never()).onError(any(Status.class)); fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); - verify(clusterWatcher).onResourceDoesNotExist("cluster-foo.googleapis.com"); + verify(cdsResourceWatcher).onResourceDoesNotExist("cluster-foo.googleapis.com"); assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); } @@ -1384,7 +1384,7 @@ public void cdsResponseWithoutMatchingResource() { */ @Test public void cdsResponseWithMatchingResource() { - xdsClient.watchClusterData("cluster-foo.googleapis.com", clusterWatcher); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", cdsResourceWatcher); StreamObserver responseObserver = responseObservers.poll(); StreamObserver requestObserver = requestObservers.poll(); @@ -1411,13 +1411,13 @@ public void cdsResponseWithMatchingResource() { XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); assertThat(cdsRespTimer.isCancelled()).isTrue(); - ArgumentCaptor clusterUpdateCaptor = ArgumentCaptor.forClass(null); - verify(clusterWatcher).onClusterChanged(clusterUpdateCaptor.capture()); - ClusterUpdate clusterUpdate = clusterUpdateCaptor.getValue(); - assertThat(clusterUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(clusterUpdate.getEdsServiceName()).isNull(); - assertThat(clusterUpdate.getLbPolicy()).isEqualTo("round_robin"); - assertThat(clusterUpdate.getLrsServerName()).isNull(); + ArgumentCaptor cdsUpdateCaptor = ArgumentCaptor.forClass(null); + verify(cdsResourceWatcher).onChanged(cdsUpdateCaptor.capture()); + CdsUpdate cdsUpdate = cdsUpdateCaptor.getValue(); + assertThat(cdsUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(cdsUpdate.getEdsServiceName()).isNull(); + assertThat(cdsUpdate.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate.getLrsServerName()).isNull(); // Management server sends back another CDS response updating the requested Cluster. clusters = ImmutableList.of( @@ -1434,13 +1434,13 @@ public void cdsResponseWithMatchingResource() { .onNext(eq(buildDiscoveryRequestV2(NODE, "1", "cluster-foo.googleapis.com", XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"))); - verify(clusterWatcher, times(2)).onClusterChanged(clusterUpdateCaptor.capture()); - clusterUpdate = clusterUpdateCaptor.getValue(); - assertThat(clusterUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(clusterUpdate.getEdsServiceName()) + verify(cdsResourceWatcher, times(2)).onChanged(cdsUpdateCaptor.capture()); + cdsUpdate = cdsUpdateCaptor.getValue(); + assertThat(cdsUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(cdsUpdate.getEdsServiceName()) .isEqualTo("eds-cluster-foo.googleapis.com"); - assertThat(clusterUpdate.getLbPolicy()).isEqualTo("round_robin"); - assertThat(clusterUpdate.getLrsServerName()).isEqualTo(""); + assertThat(cdsUpdate.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate.getLrsServerName()).isEqualTo(""); } /** @@ -1448,7 +1448,7 @@ public void cdsResponseWithMatchingResource() { */ @Test public void cdsResponseWithUpstreamTlsContext() { - xdsClient.watchClusterData("cluster-foo.googleapis.com", clusterWatcher); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", cdsResourceWatcher); StreamObserver responseObserver = responseObservers.poll(); StreamObserver requestObserver = requestObservers.poll(); @@ -1468,10 +1468,10 @@ public void cdsResponseWithUpstreamTlsContext() { verify(requestObserver) .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); - ArgumentCaptor clusterUpdateCaptor = ArgumentCaptor.forClass(null); - verify(clusterWatcher, times(1)).onClusterChanged(clusterUpdateCaptor.capture()); - ClusterUpdate clusterUpdate = clusterUpdateCaptor.getValue(); - EnvoyServerProtoData.UpstreamTlsContext upstreamTlsContext = clusterUpdate + ArgumentCaptor cdsUpdateCaptor = ArgumentCaptor.forClass(null); + verify(cdsResourceWatcher, times(1)).onChanged(cdsUpdateCaptor.capture()); + CdsUpdate cdsUpdate = cdsUpdateCaptor.getValue(); + EnvoyServerProtoData.UpstreamTlsContext upstreamTlsContext = cdsUpdate .getUpstreamTlsContext(); SdsSecretConfig validationContextSdsSecretConfig = upstreamTlsContext.getCommonTlsContext() .getValidationContextSdsSecretConfig(); @@ -1488,13 +1488,13 @@ public void cdsResponseWithUpstreamTlsContext() { } @Test - public void multipleClusterWatchers() { - ClusterWatcher watcher1 = mock(ClusterWatcher.class); - ClusterWatcher watcher2 = mock(ClusterWatcher.class); - ClusterWatcher watcher3 = mock(ClusterWatcher.class); - xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher1); - xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher2); - xdsClient.watchClusterData("cluster-bar.googleapis.com", watcher3); + public void multipleCdsWatchers() { + CdsResourceWatcher watcher1 = mock(CdsResourceWatcher.class); + CdsResourceWatcher watcher2 = mock(CdsResourceWatcher.class); + CdsResourceWatcher watcher3 = mock(CdsResourceWatcher.class); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher1); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher2); + xdsClient.watchCdsResource("cluster-bar.googleapis.com", watcher3); StreamObserver responseObserver = responseObservers.poll(); StreamObserver requestObserver = requestObservers.poll(); @@ -1526,25 +1526,25 @@ public void multipleClusterWatchers() { XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); // Two watchers get notification of cluster update for the cluster they are interested in. - ArgumentCaptor clusterUpdateCaptor1 = ArgumentCaptor.forClass(null); - verify(watcher1).onClusterChanged(clusterUpdateCaptor1.capture()); - ClusterUpdate clusterUpdate1 = clusterUpdateCaptor1.getValue(); - assertThat(clusterUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(clusterUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(clusterUpdate1.getEdsServiceName()).isNull(); - assertThat(clusterUpdate1.getLbPolicy()).isEqualTo("round_robin"); - assertThat(clusterUpdate1.getLrsServerName()).isNull(); - - ArgumentCaptor clusterUpdateCaptor2 = ArgumentCaptor.forClass(null); - verify(watcher2).onClusterChanged(clusterUpdateCaptor2.capture()); - ClusterUpdate clusterUpdate2 = clusterUpdateCaptor2.getValue(); - assertThat(clusterUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(clusterUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(clusterUpdate2.getEdsServiceName()).isNull(); - assertThat(clusterUpdate2.getLbPolicy()).isEqualTo("round_robin"); - assertThat(clusterUpdate2.getLrsServerName()).isNull(); - - verify(watcher3, never()).onClusterChanged(any(ClusterUpdate.class)); + ArgumentCaptor cdsUpdateCaptor1 = ArgumentCaptor.forClass(null); + verify(watcher1).onChanged(cdsUpdateCaptor1.capture()); + CdsUpdate cdsUpdate1 = cdsUpdateCaptor1.getValue(); + assertThat(cdsUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(cdsUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(cdsUpdate1.getEdsServiceName()).isNull(); + assertThat(cdsUpdate1.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate1.getLrsServerName()).isNull(); + + ArgumentCaptor cdsUpdateCaptor2 = ArgumentCaptor.forClass(null); + verify(watcher2).onChanged(cdsUpdateCaptor2.capture()); + CdsUpdate cdsUpdate2 = cdsUpdateCaptor2.getValue(); + assertThat(cdsUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(cdsUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(cdsUpdate2.getEdsServiceName()).isNull(); + assertThat(cdsUpdate2.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate2.getLrsServerName()).isNull(); + + verify(watcher3, never()).onChanged(any(CdsUpdate.class)); verify(watcher3, never()).onResourceDoesNotExist("cluster-bar.googleapis.com"); verify(watcher3, never()).onError(any(Status.class)); @@ -1573,14 +1573,14 @@ public void multipleClusterWatchers() { XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"))); verifyNoMoreInteractions(watcher1, watcher2); // resource has no change - ArgumentCaptor clusterUpdateCaptor3 = ArgumentCaptor.forClass(null); - verify(watcher3).onClusterChanged(clusterUpdateCaptor3.capture()); - ClusterUpdate clusterUpdate3 = clusterUpdateCaptor3.getValue(); - assertThat(clusterUpdate3.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); - assertThat(clusterUpdate3.getEdsServiceName()) + ArgumentCaptor cdsUpdateCaptor3 = ArgumentCaptor.forClass(null); + verify(watcher3).onChanged(cdsUpdateCaptor3.capture()); + CdsUpdate cdsUpdate3 = cdsUpdateCaptor3.getValue(); + assertThat(cdsUpdate3.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); + assertThat(cdsUpdate3.getEdsServiceName()) .isEqualTo("eds-cluster-bar.googleapis.com"); - assertThat(clusterUpdate3.getLbPolicy()).isEqualTo("round_robin"); - assertThat(clusterUpdate3.getLrsServerName()).isEqualTo(""); + assertThat(cdsUpdate3.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate3.getLrsServerName()).isEqualTo(""); } /** @@ -1590,8 +1590,8 @@ public void multipleClusterWatchers() { */ @Test public void watchClusterAlreadyBeingWatched() { - ClusterWatcher watcher1 = mock(ClusterWatcher.class); - xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher1); + CdsResourceWatcher watcher1 = mock(CdsResourceWatcher.class); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher1); // Streaming RPC starts after a first watcher is added. StreamObserver responseObserver = responseObservers.poll(); @@ -1616,28 +1616,28 @@ public void watchClusterAlreadyBeingWatched() { .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); - ArgumentCaptor clusterUpdateCaptor1 = ArgumentCaptor.forClass(null); - verify(watcher1).onClusterChanged(clusterUpdateCaptor1.capture()); - ClusterUpdate clusterUpdate1 = clusterUpdateCaptor1.getValue(); - assertThat(clusterUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(clusterUpdate1.getEdsServiceName()).isNull(); - assertThat(clusterUpdate1.getLbPolicy()).isEqualTo("round_robin"); - assertThat(clusterUpdate1.getLrsServerName()).isNull(); + ArgumentCaptor cdsUpdateCaptor1 = ArgumentCaptor.forClass(null); + verify(watcher1).onChanged(cdsUpdateCaptor1.capture()); + CdsUpdate cdsUpdate1 = cdsUpdateCaptor1.getValue(); + assertThat(cdsUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(cdsUpdate1.getEdsServiceName()).isNull(); + assertThat(cdsUpdate1.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate1.getLrsServerName()).isNull(); assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); // Another cluster watcher interested in the same cluster is added. - ClusterWatcher watcher2 = mock(ClusterWatcher.class); - xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher2); + CdsResourceWatcher watcher2 = mock(CdsResourceWatcher.class); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher2); // Since the client has received cluster update for this cluster before, cached result is // notified to the newly added watcher immediately. - ArgumentCaptor clusterUpdateCaptor2 = ArgumentCaptor.forClass(null); - verify(watcher2).onClusterChanged(clusterUpdateCaptor2.capture()); - ClusterUpdate clusterUpdate2 = clusterUpdateCaptor2.getValue(); - assertThat(clusterUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(clusterUpdate2.getEdsServiceName()).isNull(); - assertThat(clusterUpdate2.getLbPolicy()).isEqualTo("round_robin"); - assertThat(clusterUpdate2.getLrsServerName()).isNull(); + ArgumentCaptor cdsUpdateCaptor2 = ArgumentCaptor.forClass(null); + verify(watcher2).onChanged(cdsUpdateCaptor2.capture()); + CdsUpdate cdsUpdate2 = cdsUpdateCaptor2.getValue(); + assertThat(cdsUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(cdsUpdate2.getEdsServiceName()).isNull(); + assertThat(cdsUpdate2.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate2.getLrsServerName()).isNull(); verifyNoMoreInteractions(requestObserver); assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); @@ -1647,9 +1647,9 @@ public void watchClusterAlreadyBeingWatched() { * Basic operations of adding/canceling cluster data watchers. */ @Test - public void addRemoveClusterWatchers() { - ClusterWatcher watcher1 = mock(ClusterWatcher.class); - xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher1); + public void addRemoveCdsWatchers() { + CdsResourceWatcher watcher1 = mock(CdsResourceWatcher.class); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher1); // Streaming RPC starts after a first watcher is added. StreamObserver responseObserver = responseObservers.poll(); @@ -1673,17 +1673,17 @@ public void addRemoveClusterWatchers() { .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); - ArgumentCaptor clusterUpdateCaptor1 = ArgumentCaptor.forClass(null); - verify(watcher1).onClusterChanged(clusterUpdateCaptor1.capture()); - ClusterUpdate clusterUpdate1 = clusterUpdateCaptor1.getValue(); - assertThat(clusterUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(clusterUpdate1.getEdsServiceName()).isNull(); - assertThat(clusterUpdate1.getLbPolicy()).isEqualTo("round_robin"); - assertThat(clusterUpdate1.getLrsServerName()).isNull(); + ArgumentCaptor cdsUpdateCaptor1 = ArgumentCaptor.forClass(null); + verify(watcher1).onChanged(cdsUpdateCaptor1.capture()); + CdsUpdate cdsUpdate1 = cdsUpdateCaptor1.getValue(); + assertThat(cdsUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(cdsUpdate1.getEdsServiceName()).isNull(); + assertThat(cdsUpdate1.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate1.getLrsServerName()).isNull(); // Add another cluster watcher for a different cluster. - ClusterWatcher watcher2 = mock(ClusterWatcher.class); - xdsClient.watchClusterData("cluster-bar.googleapis.com", watcher2); + CdsResourceWatcher watcher2 = mock(CdsResourceWatcher.class); + xdsClient.watchCdsResource("cluster-bar.googleapis.com", watcher2); // Client sent a new CDS request for all interested resources. verify(requestObserver) @@ -1711,17 +1711,17 @@ public void addRemoveClusterWatchers() { ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"))); verifyNoMoreInteractions(watcher1); // resource has no change - ArgumentCaptor clusterUpdateCaptor2 = ArgumentCaptor.forClass(null); - verify(watcher2).onClusterChanged(clusterUpdateCaptor2.capture()); - ClusterUpdate clusterUpdate2 = clusterUpdateCaptor2.getValue(); - assertThat(clusterUpdate2.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); - assertThat(clusterUpdate2.getEdsServiceName()) + ArgumentCaptor cdsUpdateCaptor2 = ArgumentCaptor.forClass(null); + verify(watcher2).onChanged(cdsUpdateCaptor2.capture()); + CdsUpdate cdsUpdate2 = cdsUpdateCaptor2.getValue(); + assertThat(cdsUpdate2.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); + assertThat(cdsUpdate2.getEdsServiceName()) .isEqualTo("eds-cluster-bar.googleapis.com"); - assertThat(clusterUpdate2.getLbPolicy()).isEqualTo("round_robin"); - assertThat(clusterUpdate2.getLrsServerName()).isEqualTo(""); + assertThat(cdsUpdate2.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate2.getLrsServerName()).isEqualTo(""); // Cancel one of the watcher. - xdsClient.cancelClusterDataWatch("cluster-foo.googleapis.com", watcher1); + xdsClient.cancelCdsResourceWatch("cluster-foo.googleapis.com", watcher1); // Since the cancelled watcher was the last watcher interested in that cluster (but there // is still interested resource), client sent an new CDS request to unsubscribe from @@ -1733,7 +1733,7 @@ public void addRemoveClusterWatchers() { // Management server has nothing to respond. // Cancel the other watcher. All resources have been unsubscribed. - xdsClient.cancelClusterDataWatch("cluster-bar.googleapis.com", watcher2); + xdsClient.cancelCdsResourceWatch("cluster-bar.googleapis.com", watcher2); verify(requestObserver) .onNext( @@ -1760,9 +1760,9 @@ public void addRemoveClusterWatchers() { verifyNoMoreInteractions(watcher1, watcher2); // A new cluster watcher is added to watch cluster foo again. - ClusterWatcher watcher3 = mock(ClusterWatcher.class); - xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher3); - verify(watcher3, never()).onClusterChanged(any(ClusterUpdate.class)); + CdsResourceWatcher watcher3 = mock(CdsResourceWatcher.class); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher3); + verify(watcher3, never()).onChanged(any(CdsUpdate.class)); // A CDS request is sent to indicate subscription of "cluster-foo.googleapis.com" only. verify(requestObserver) @@ -1780,13 +1780,13 @@ public void addRemoveClusterWatchers() { responseObserver.onNext(response); // Notified with cached data immediately. - ArgumentCaptor clusterUpdateCaptor3 = ArgumentCaptor.forClass(null); - verify(watcher3).onClusterChanged(clusterUpdateCaptor3.capture()); - ClusterUpdate clusterUpdate3 = clusterUpdateCaptor3.getValue(); - assertThat(clusterUpdate3.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(clusterUpdate3.getEdsServiceName()).isNull(); - assertThat(clusterUpdate3.getLbPolicy()).isEqualTo("round_robin"); - assertThat(clusterUpdate2.getLrsServerName()).isEqualTo(""); + ArgumentCaptor cdsUpdateCaptor3 = ArgumentCaptor.forClass(null); + verify(watcher3).onChanged(cdsUpdateCaptor3.capture()); + CdsUpdate cdsUpdate3 = cdsUpdateCaptor3.getValue(); + assertThat(cdsUpdate3.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(cdsUpdate3.getEdsServiceName()).isNull(); + assertThat(cdsUpdate3.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate2.getLrsServerName()).isEqualTo(""); verifyNoMoreInteractions(watcher1, watcher2); @@ -1797,9 +1797,9 @@ public void addRemoveClusterWatchers() { } @Test - public void addRemoveClusterWatcherWhileInitialResourceFetchInProgress() { - ClusterWatcher watcher1 = mock(ClusterWatcher.class); - xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher1); + public void addRemoveCdsWatcherWhileInitialResourceFetchInProgress() { + CdsResourceWatcher watcher1 = mock(CdsResourceWatcher.class); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher1); // Streaming RPC starts after a first watcher is added. StreamObserver requestObserver = requestObservers.poll(); @@ -1814,12 +1814,12 @@ public void addRemoveClusterWatcherWhileInitialResourceFetchInProgress() { fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC - 1, TimeUnit.SECONDS); - ClusterWatcher watcher2 = mock(ClusterWatcher.class); - ClusterWatcher watcher3 = mock(ClusterWatcher.class); - ClusterWatcher watcher4 = mock(ClusterWatcher.class); - xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher2); - xdsClient.watchClusterData("cluster-bar.googleapis.com", watcher3); - xdsClient.watchClusterData("cluster-bar.googleapis.com", watcher4); + CdsResourceWatcher watcher2 = mock(CdsResourceWatcher.class); + CdsResourceWatcher watcher3 = mock(CdsResourceWatcher.class); + CdsResourceWatcher watcher4 = mock(CdsResourceWatcher.class); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher2); + xdsClient.watchCdsResource("cluster-bar.googleapis.com", watcher3); + xdsClient.watchCdsResource("cluster-bar.googleapis.com", watcher4); // Client sends a new CDS request for updating the latest resource subscription. verify(requestObserver) @@ -1838,8 +1838,8 @@ public void addRemoveClusterWatcherWhileInitialResourceFetchInProgress() { verify(watcher2).onResourceDoesNotExist("cluster-foo.googleapis.com"); // The absence result is known immediately. - ClusterWatcher watcher5 = mock(ClusterWatcher.class); - xdsClient.watchClusterData("cluster-foo.googleapis.com", watcher5); + CdsResourceWatcher watcher5 = mock(CdsResourceWatcher.class); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher5); verify(watcher5).onResourceDoesNotExist("cluster-foo.googleapis.com"); assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); @@ -1847,9 +1847,9 @@ public void addRemoveClusterWatcherWhileInitialResourceFetchInProgress() { // Cancel watchers while discovery for resource "cluster-bar.googleapis.com" is still // in progress. - xdsClient.cancelClusterDataWatch("cluster-bar.googleapis.com", watcher3); + xdsClient.cancelCdsResourceWatch("cluster-bar.googleapis.com", watcher3); assertThat(timeoutTask.isCancelled()).isFalse(); - xdsClient.cancelClusterDataWatch("cluster-bar.googleapis.com", watcher4); + xdsClient.cancelCdsResourceWatch("cluster-bar.googleapis.com", watcher4); // Client sends a CDS request for resource subscription update (Omitted). @@ -1863,7 +1863,7 @@ public void addRemoveClusterWatcherWhileInitialResourceFetchInProgress() { @Test public void cdsUpdateForClusterBeingRemoved() { - xdsClient.watchClusterData("cluster-foo.googleapis.com", clusterWatcher); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", cdsResourceWatcher); StreamObserver responseObserver = responseObservers.poll(); StreamObserver requestObserver = requestObservers.poll(); @@ -1881,13 +1881,13 @@ public void cdsUpdateForClusterBeingRemoved() { // Client sent an ACK CDS request (Omitted). - ArgumentCaptor clusterUpdateCaptor = ArgumentCaptor.forClass(null); - verify(clusterWatcher).onClusterChanged(clusterUpdateCaptor.capture()); - ClusterUpdate clusterUpdate = clusterUpdateCaptor.getValue(); - assertThat(clusterUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(clusterUpdate.getEdsServiceName()).isNull(); - assertThat(clusterUpdate.getLbPolicy()).isEqualTo("round_robin"); - assertThat(clusterUpdate.getLrsServerName()).isEqualTo(""); + ArgumentCaptor cdsUpdateCaptor = ArgumentCaptor.forClass(null); + verify(cdsResourceWatcher).onChanged(cdsUpdateCaptor.capture()); + CdsUpdate cdsUpdate = cdsUpdateCaptor.getValue(); + assertThat(cdsUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(cdsUpdate.getEdsServiceName()).isNull(); + assertThat(cdsUpdate.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate.getLrsServerName()).isEqualTo(""); assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); // No cluster is available. @@ -1896,7 +1896,7 @@ public void cdsUpdateForClusterBeingRemoved() { XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"); responseObserver.onNext(response); - verify(clusterWatcher).onResourceDoesNotExist("cluster-foo.googleapis.com"); + verify(cdsResourceWatcher).onResourceDoesNotExist("cluster-foo.googleapis.com"); } /** @@ -1908,7 +1908,7 @@ public void cdsUpdateForClusterBeingRemoved() { */ @Test public void edsResponseWithoutMatchingResource() { - xdsClient.watchEndpointData("cluster-foo.googleapis.com", endpointWatcher); + xdsClient.watchEdsResource("cluster-foo.googleapis.com", edsResourceWatcher); StreamObserver responseObserver = responseObservers.poll(); StreamObserver requestObserver = requestObservers.poll(); @@ -1946,11 +1946,11 @@ public void edsResponseWithoutMatchingResource() { .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"))); - verify(endpointWatcher, never()).onEndpointChanged(any(EndpointUpdate.class)); - verify(endpointWatcher, never()).onResourceDoesNotExist("cluster-foo.googleapis.com"); - verify(endpointWatcher, never()).onError(any(Status.class)); + verify(edsResourceWatcher, never()).onChanged(any(EdsUpdate.class)); + verify(edsResourceWatcher, never()).onResourceDoesNotExist("cluster-foo.googleapis.com"); + verify(edsResourceWatcher, never()).onError(any(Status.class)); fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); - verify(endpointWatcher).onResourceDoesNotExist("cluster-foo.googleapis.com"); + verify(edsResourceWatcher).onResourceDoesNotExist("cluster-foo.googleapis.com"); assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); } @@ -1960,7 +1960,7 @@ public void edsResponseWithoutMatchingResource() { */ @Test public void edsResponseWithMatchingResource() { - xdsClient.watchEndpointData("cluster-foo.googleapis.com", endpointWatcher); + xdsClient.watchEdsResource("cluster-foo.googleapis.com", edsResourceWatcher); StreamObserver responseObserver = responseObservers.poll(); StreamObserver requestObserver = requestObservers.poll(); @@ -2012,15 +2012,15 @@ public void edsResponseWithMatchingResource() { .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"))); - ArgumentCaptor endpointUpdateCaptor = ArgumentCaptor.forClass(null); - verify(endpointWatcher).onEndpointChanged(endpointUpdateCaptor.capture()); - EndpointUpdate endpointUpdate = endpointUpdateCaptor.getValue(); - assertThat(endpointUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(endpointUpdate.getDropPolicies()) + ArgumentCaptor edsUpdateCaptor = ArgumentCaptor.forClass(null); + verify(edsResourceWatcher).onChanged(edsUpdateCaptor.capture()); + EdsUpdate edsUpdate = edsUpdateCaptor.getValue(); + assertThat(edsUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(edsUpdate.getDropPolicies()) .containsExactly( new DropOverload("lb", 200), new DropOverload("throttle", 1000)); - assertThat(endpointUpdate.getLocalityLbEndpointsMap()) + assertThat(edsUpdate.getLocalityLbEndpointsMap()) .containsExactly( new Locality("region1", "zone1", "subzone1"), new LocalityLbEndpoints( @@ -2045,21 +2045,21 @@ public void edsResponseWithMatchingResource() { .onNext(eq(buildDiscoveryRequestV2(NODE, "1", "cluster-foo.googleapis.com", XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"))); - verify(endpointWatcher, times(2)).onEndpointChanged(endpointUpdateCaptor.capture()); - endpointUpdate = endpointUpdateCaptor.getValue(); - assertThat(endpointUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(endpointUpdate.getDropPolicies()).isEmpty(); - assertThat(endpointUpdate.getLocalityLbEndpointsMap()).isEmpty(); + verify(edsResourceWatcher, times(2)).onChanged(edsUpdateCaptor.capture()); + edsUpdate = edsUpdateCaptor.getValue(); + assertThat(edsUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(edsUpdate.getDropPolicies()).isEmpty(); + assertThat(edsUpdate.getLocalityLbEndpointsMap()).isEmpty(); } @Test - public void multipleEndpointWatchers() { - EndpointWatcher watcher1 = mock(EndpointWatcher.class); - EndpointWatcher watcher2 = mock(EndpointWatcher.class); - EndpointWatcher watcher3 = mock(EndpointWatcher.class); - xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher1); - xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher2); - xdsClient.watchEndpointData("cluster-bar.googleapis.com", watcher3); + public void multipleEdsWatchers() { + EdsResourceWatcher watcher1 = mock(EdsResourceWatcher.class); + EdsResourceWatcher watcher2 = mock(EdsResourceWatcher.class); + EdsResourceWatcher watcher3 = mock(EdsResourceWatcher.class); + xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher1); + xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher2); + xdsClient.watchEdsResource("cluster-bar.googleapis.com", watcher3); StreamObserver responseObserver = responseObservers.poll(); StreamObserver requestObserver = requestObservers.poll(); @@ -2101,11 +2101,11 @@ public void multipleEndpointWatchers() { XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"))); // Two watchers get notification of endpoint update for the cluster they are interested in. - ArgumentCaptor endpointUpdateCaptor1 = ArgumentCaptor.forClass(null); - verify(watcher1).onEndpointChanged(endpointUpdateCaptor1.capture()); - EndpointUpdate endpointUpdate1 = endpointUpdateCaptor1.getValue(); - assertThat(endpointUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(endpointUpdate1.getLocalityLbEndpointsMap()) + ArgumentCaptor edsUpdateCaptor1 = ArgumentCaptor.forClass(null); + verify(watcher1).onChanged(edsUpdateCaptor1.capture()); + EdsUpdate edsUpdate1 = edsUpdateCaptor1.getValue(); + assertThat(edsUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(edsUpdate1.getLocalityLbEndpointsMap()) .containsExactly( new Locality("region1", "zone1", "subzone1"), new LocalityLbEndpoints( @@ -2113,11 +2113,11 @@ public void multipleEndpointWatchers() { new LbEndpoint("192.168.0.1", 8080, 2, true)), 1, 0)); - ArgumentCaptor endpointUpdateCaptor2 = ArgumentCaptor.forClass(null); - verify(watcher1).onEndpointChanged(endpointUpdateCaptor2.capture()); - EndpointUpdate endpointUpdate2 = endpointUpdateCaptor2.getValue(); - assertThat(endpointUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(endpointUpdate2.getLocalityLbEndpointsMap()) + ArgumentCaptor edsUpdateCaptor2 = ArgumentCaptor.forClass(null); + verify(watcher1).onChanged(edsUpdateCaptor2.capture()); + EdsUpdate edsUpdate2 = edsUpdateCaptor2.getValue(); + assertThat(edsUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(edsUpdate2.getLocalityLbEndpointsMap()) .containsExactly( new Locality("region1", "zone1", "subzone1"), new LocalityLbEndpoints( @@ -2151,11 +2151,11 @@ public void multipleEndpointWatchers() { XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"))); // The corresponding watcher gets notified. - ArgumentCaptor endpointUpdateCaptor3 = ArgumentCaptor.forClass(null); - verify(watcher3).onEndpointChanged(endpointUpdateCaptor3.capture()); - EndpointUpdate endpointUpdate3 = endpointUpdateCaptor3.getValue(); - assertThat(endpointUpdate3.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); - assertThat(endpointUpdate3.getLocalityLbEndpointsMap()) + ArgumentCaptor edsUpdateCaptor3 = ArgumentCaptor.forClass(null); + verify(watcher3).onChanged(edsUpdateCaptor3.capture()); + EdsUpdate edsUpdate3 = edsUpdateCaptor3.getValue(); + assertThat(edsUpdate3.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); + assertThat(edsUpdate3.getLocalityLbEndpointsMap()) .containsExactly( new Locality("region2", "zone2", "subzone2"), new LocalityLbEndpoints( @@ -2171,8 +2171,8 @@ public void multipleEndpointWatchers() { */ @Test public void watchEndpointsForClusterAlreadyBeingWatched() { - EndpointWatcher watcher1 = mock(EndpointWatcher.class); - xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher1); + EdsResourceWatcher watcher1 = mock(EdsResourceWatcher.class); + xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher1); StreamObserver responseObserver = responseObservers.poll(); StreamObserver requestObserver = requestObservers.poll(); @@ -2206,12 +2206,12 @@ public void watchEndpointsForClusterAlreadyBeingWatched() { .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"))); - ArgumentCaptor endpointUpdateCaptor1 = ArgumentCaptor.forClass(null); - verify(watcher1).onEndpointChanged(endpointUpdateCaptor1.capture()); - EndpointUpdate endpointUpdate1 = endpointUpdateCaptor1.getValue(); - assertThat(endpointUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(endpointUpdate1.getDropPolicies()).isEmpty(); - assertThat(endpointUpdate1.getLocalityLbEndpointsMap()) + ArgumentCaptor edsUpdateCaptor1 = ArgumentCaptor.forClass(null); + verify(watcher1).onChanged(edsUpdateCaptor1.capture()); + EdsUpdate edsUpdate1 = edsUpdateCaptor1.getValue(); + assertThat(edsUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(edsUpdate1.getDropPolicies()).isEmpty(); + assertThat(edsUpdate1.getLocalityLbEndpointsMap()) .containsExactly( new Locality("region1", "zone1", "subzone1"), new LocalityLbEndpoints( @@ -2220,17 +2220,17 @@ public void watchEndpointsForClusterAlreadyBeingWatched() { 2, true)), 1, 0)); // A second endpoint watcher is registered for endpoints in the same cluster. - EndpointWatcher watcher2 = mock(EndpointWatcher.class); - xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher2); + EdsResourceWatcher watcher2 = mock(EdsResourceWatcher.class); + xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher2); // Cached endpoint information is notified to the new watcher immediately, without sending // another EDS request. - ArgumentCaptor endpointUpdateCaptor2 = ArgumentCaptor.forClass(null); - verify(watcher2).onEndpointChanged(endpointUpdateCaptor2.capture()); - EndpointUpdate endpointUpdate2 = endpointUpdateCaptor2.getValue(); - assertThat(endpointUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(endpointUpdate2.getDropPolicies()).isEmpty(); - assertThat(endpointUpdate2.getLocalityLbEndpointsMap()) + ArgumentCaptor edsUpdateCaptor2 = ArgumentCaptor.forClass(null); + verify(watcher2).onChanged(edsUpdateCaptor2.capture()); + EdsUpdate edsUpdate2 = edsUpdateCaptor2.getValue(); + assertThat(edsUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(edsUpdate2.getDropPolicies()).isEmpty(); + assertThat(edsUpdate2.getLocalityLbEndpointsMap()) .containsExactly( new Locality("region1", "zone1", "subzone1"), new LocalityLbEndpoints( @@ -2246,9 +2246,9 @@ public void watchEndpointsForClusterAlreadyBeingWatched() { * Basic operations of adding/canceling endpoint data watchers. */ @Test - public void addRemoveEndpointWatchers() { - EndpointWatcher watcher1 = mock(EndpointWatcher.class); - xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher1); + public void addRemoveEdsWatchers() { + EdsResourceWatcher watcher1 = mock(EdsResourceWatcher.class); + xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher1); // Streaming RPC starts after a first watcher is added. StreamObserver responseObserver = responseObservers.poll(); @@ -2281,11 +2281,11 @@ public void addRemoveEndpointWatchers() { .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"))); - ArgumentCaptor endpointUpdateCaptor1 = ArgumentCaptor.forClass(null); - verify(watcher1).onEndpointChanged(endpointUpdateCaptor1.capture()); - EndpointUpdate endpointUpdate1 = endpointUpdateCaptor1.getValue(); - assertThat(endpointUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(endpointUpdate1.getLocalityLbEndpointsMap()) + ArgumentCaptor edsUpdateCaptor1 = ArgumentCaptor.forClass(null); + verify(watcher1).onChanged(edsUpdateCaptor1.capture()); + EdsUpdate edsUpdate1 = edsUpdateCaptor1.getValue(); + assertThat(edsUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); + assertThat(edsUpdate1.getLocalityLbEndpointsMap()) .containsExactly( new Locality("region1", "zone1", "subzone1"), new LocalityLbEndpoints( @@ -2295,8 +2295,8 @@ public void addRemoveEndpointWatchers() { 1, 0)); // Add another endpoint watcher for a different cluster. - EndpointWatcher watcher2 = mock(EndpointWatcher.class); - xdsClient.watchEndpointData("cluster-bar.googleapis.com", watcher2); + EdsResourceWatcher watcher2 = mock(EdsResourceWatcher.class); + xdsClient.watchEdsResource("cluster-bar.googleapis.com", watcher2); // Client sent a new EDS request for all interested resources. verify(requestObserver) @@ -2329,11 +2329,11 @@ public void addRemoveEndpointWatchers() { ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"))); - ArgumentCaptor endpointUpdateCaptor2 = ArgumentCaptor.forClass(null); - verify(watcher2).onEndpointChanged(endpointUpdateCaptor2.capture()); - EndpointUpdate endpointUpdate2 = endpointUpdateCaptor2.getValue(); - assertThat(endpointUpdate2.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); - assertThat(endpointUpdate2.getLocalityLbEndpointsMap()) + ArgumentCaptor edsUpdateCaptor2 = ArgumentCaptor.forClass(null); + verify(watcher2).onChanged(edsUpdateCaptor2.capture()); + EdsUpdate edsUpdate2 = edsUpdateCaptor2.getValue(); + assertThat(edsUpdate2.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); + assertThat(edsUpdate2.getLocalityLbEndpointsMap()) .containsExactly( new Locality("region2", "zone2", "subzone2"), new LocalityLbEndpoints( @@ -2342,7 +2342,7 @@ public void addRemoveEndpointWatchers() { 6, 0)); // Cancel one of the watcher. - xdsClient.cancelEndpointDataWatch("cluster-foo.googleapis.com", watcher1); + xdsClient.cancelEdsResourceWatch("cluster-foo.googleapis.com", watcher1); // Since the cancelled watcher was the last watcher interested in that cluster, client // sent an new EDS request to unsubscribe from that cluster. @@ -2353,7 +2353,7 @@ public void addRemoveEndpointWatchers() { // Management server should not respond as it had previously sent the requested resource. // Cancel the other watcher. - xdsClient.cancelEndpointDataWatch("cluster-bar.googleapis.com", watcher2); + xdsClient.cancelEdsResourceWatch("cluster-bar.googleapis.com", watcher2); // Since the cancelled watcher was the last watcher interested in that cluster, client // sent an new EDS request to unsubscribe from that cluster. @@ -2399,13 +2399,13 @@ public void addRemoveEndpointWatchers() { verifyNoMoreInteractions(watcher1, watcher2); // A new endpoint watcher is added to watch an old but was no longer interested in cluster. - EndpointWatcher watcher3 = mock(EndpointWatcher.class); - xdsClient.watchEndpointData("cluster-bar.googleapis.com", watcher3); + EdsResourceWatcher watcher3 = mock(EdsResourceWatcher.class); + xdsClient.watchEdsResource("cluster-bar.googleapis.com", watcher3); // Nothing should be notified to the new watcher as we are still waiting management server's // latest response. // Cached endpoint data should have been purged. - verify(watcher3, never()).onEndpointChanged(any(EndpointUpdate.class)); + verify(watcher3, never()).onChanged(any(EdsUpdate.class)); // An EDS request is sent to re-subscribe the cluster again. verify(requestObserver) @@ -2426,11 +2426,11 @@ public void addRemoveEndpointWatchers() { XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0003"); responseObserver.onNext(response); - ArgumentCaptor endpointUpdateCaptor3 = ArgumentCaptor.forClass(null); - verify(watcher3).onEndpointChanged(endpointUpdateCaptor3.capture()); - EndpointUpdate endpointUpdate3 = endpointUpdateCaptor3.getValue(); - assertThat(endpointUpdate3.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); - assertThat(endpointUpdate3.getLocalityLbEndpointsMap()) + ArgumentCaptor edsUpdateCaptor3 = ArgumentCaptor.forClass(null); + verify(watcher3).onChanged(edsUpdateCaptor3.capture()); + EdsUpdate edsUpdate3 = edsUpdateCaptor3.getValue(); + assertThat(edsUpdate3.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); + assertThat(edsUpdate3.getLocalityLbEndpointsMap()) .containsExactly( new Locality("region4", "zone4", "subzone4"), new LocalityLbEndpoints( @@ -2448,9 +2448,9 @@ public void addRemoveEndpointWatchers() { } @Test - public void addRemoveEndpointWatcherWhileInitialResourceFetchInProgress() { - EndpointWatcher watcher1 = mock(EndpointWatcher.class); - xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher1); + public void addRemoveEdsWatcherWhileInitialResourceFetchInProgress() { + EdsResourceWatcher watcher1 = mock(EdsResourceWatcher.class); + xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher1); // Streaming RPC starts after a first watcher is added. StreamObserver requestObserver = requestObservers.poll(); @@ -2465,12 +2465,12 @@ public void addRemoveEndpointWatcherWhileInitialResourceFetchInProgress() { fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC - 1, TimeUnit.SECONDS); - EndpointWatcher watcher2 = mock(EndpointWatcher.class); - EndpointWatcher watcher3 = mock(EndpointWatcher.class); - EndpointWatcher watcher4 = mock(EndpointWatcher.class); - xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher2); - xdsClient.watchEndpointData("cluster-bar.googleapis.com", watcher3); - xdsClient.watchEndpointData("cluster-bar.googleapis.com", watcher4); + EdsResourceWatcher watcher2 = mock(EdsResourceWatcher.class); + EdsResourceWatcher watcher3 = mock(EdsResourceWatcher.class); + EdsResourceWatcher watcher4 = mock(EdsResourceWatcher.class); + xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher2); + xdsClient.watchEdsResource("cluster-bar.googleapis.com", watcher3); + xdsClient.watchEdsResource("cluster-bar.googleapis.com", watcher4); // Client sends a new EDS request for updating the latest resource subscription. verify(requestObserver) @@ -2489,8 +2489,8 @@ public void addRemoveEndpointWatcherWhileInitialResourceFetchInProgress() { verify(watcher2).onResourceDoesNotExist("cluster-foo.googleapis.com"); // The absence result is known immediately. - EndpointWatcher watcher5 = mock(EndpointWatcher.class); - xdsClient.watchEndpointData("cluster-foo.googleapis.com", watcher5); + EdsResourceWatcher watcher5 = mock(EdsResourceWatcher.class); + xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher5); verify(watcher5).onResourceDoesNotExist("cluster-foo.googleapis.com"); assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); @@ -2498,9 +2498,9 @@ public void addRemoveEndpointWatcherWhileInitialResourceFetchInProgress() { // Cancel watchers while discovery for resource "cluster-bar.googleapis.com" is still // in progress. - xdsClient.cancelEndpointDataWatch("cluster-bar.googleapis.com", watcher3); + xdsClient.cancelEdsResourceWatch("cluster-bar.googleapis.com", watcher3); assertThat(timeoutTask.isCancelled()).isFalse(); - xdsClient.cancelEndpointDataWatch("cluster-bar.googleapis.com", watcher4); + xdsClient.cancelEdsResourceWatch("cluster-bar.googleapis.com", watcher4); // Client sends an EDS request for resource subscription update (Omitted). @@ -2514,7 +2514,7 @@ public void addRemoveEndpointWatcherWhileInitialResourceFetchInProgress() { @Test public void cdsUpdateForEdsServiceNameChange() { - xdsClient.watchClusterData("cluster-foo.googleapis.com", clusterWatcher); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", cdsResourceWatcher); StreamObserver responseObserver = responseObservers.poll(); // Management server sends back a CDS response containing requested resource. @@ -2524,7 +2524,7 @@ public void cdsUpdateForEdsServiceNameChange() { buildDiscoveryResponseV2("0", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"); responseObserver.onNext(response); - xdsClient.watchEndpointData("cluster-foo:service-bar", endpointWatcher); + xdsClient.watchEdsResource("cluster-foo:service-bar", edsResourceWatcher); // Management server sends back an EDS response for resource "cluster-foo:service-bar". List clusterLoadAssignments = ImmutableList.of( @@ -2540,12 +2540,12 @@ public void cdsUpdateForEdsServiceNameChange() { XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"); responseObserver.onNext(response); - ArgumentCaptor endpointUpdateCaptor = ArgumentCaptor.forClass(null); - verify(endpointWatcher).onEndpointChanged(endpointUpdateCaptor.capture()); - EndpointUpdate endpointUpdate = endpointUpdateCaptor.getValue(); - assertThat(endpointUpdate.getClusterName()).isEqualTo("cluster-foo:service-bar"); - assertThat(endpointUpdate.getDropPolicies()).isEmpty(); - assertThat(endpointUpdate.getLocalityLbEndpointsMap()) + ArgumentCaptor edsUpdateCaptor = ArgumentCaptor.forClass(null); + verify(edsResourceWatcher).onChanged(edsUpdateCaptor.capture()); + EdsUpdate edsUpdate = edsUpdateCaptor.getValue(); + assertThat(edsUpdate.getClusterName()).isEqualTo("cluster-foo:service-bar"); + assertThat(edsUpdate.getDropPolicies()).isEmpty(); + assertThat(edsUpdate.getLocalityLbEndpointsMap()) .containsExactly( new Locality("region1", "zone1", "subzone1"), new LocalityLbEndpoints( @@ -2562,7 +2562,7 @@ public void cdsUpdateForEdsServiceNameChange() { responseObserver.onNext(response); // Watcher get notification for endpoint resource "cluster-foo:service-bar" being deleted. - verify(endpointWatcher).onResourceDoesNotExist("cluster-foo:service-bar"); + verify(edsResourceWatcher).onResourceDoesNotExist("cluster-foo:service-bar"); } /** @@ -2748,7 +2748,7 @@ public void streamClosedAndRetry() { ArgumentCaptor statusCaptor = ArgumentCaptor.forClass(null); // Start watching cluster information. - xdsClient.watchClusterData("cluster.googleapis.com", clusterWatcher); + xdsClient.watchCdsResource("cluster.googleapis.com", cdsResourceWatcher); // Client sent first CDS request. verify(requestObserver) @@ -2756,7 +2756,7 @@ public void streamClosedAndRetry() { XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); // Start watching endpoint information. - xdsClient.watchEndpointData("cluster.googleapis.com", endpointWatcher); + xdsClient.watchEdsResource("cluster.googleapis.com", edsResourceWatcher); // Client sent first EDS request. verify(requestObserver) @@ -2767,9 +2767,9 @@ public void streamClosedAndRetry() { responseObserver.onError(Status.UNKNOWN.asException()); verify(configWatcher).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNKNOWN); - verify(clusterWatcher).onError(statusCaptor.capture()); + verify(cdsResourceWatcher).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNKNOWN); - verify(endpointWatcher).onError(statusCaptor.capture()); + verify(edsResourceWatcher).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNKNOWN); // Resets backoff and retry immediately. @@ -2795,9 +2795,9 @@ public void streamClosedAndRetry() { responseObserver.onError(Status.UNAVAILABLE.asException()); verify(configWatcher, times(2)).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); - verify(clusterWatcher, times(2)).onError(statusCaptor.capture()); + verify(cdsResourceWatcher, times(2)).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); - verify(endpointWatcher, times(2)).onError(statusCaptor.capture()); + verify(edsResourceWatcher, times(2)).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); inOrder.verify(backoffPolicy1).nextBackoffNanos(); assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); @@ -2824,9 +2824,9 @@ public void streamClosedAndRetry() { responseObserver.onError(Status.UNAVAILABLE.asException()); verify(configWatcher, times(3)).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); - verify(clusterWatcher, times(3)).onError(statusCaptor.capture()); + verify(cdsResourceWatcher, times(3)).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); - verify(endpointWatcher, times(3)).onError(statusCaptor.capture()); + verify(edsResourceWatcher, times(3)).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); inOrder.verify(backoffPolicy1).nextBackoffNanos(); assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); @@ -2861,8 +2861,8 @@ public void streamClosedAndRetry() { // Management server closes the RPC stream. responseObserver.onCompleted(); verify(configWatcher, times(4)).onError(any(Status.class)); - verify(clusterWatcher, times(4)).onError(any(Status.class)); - verify(endpointWatcher, times(4)).onError(any(Status.class)); + verify(cdsResourceWatcher, times(4)).onError(any(Status.class)); + verify(edsResourceWatcher, times(4)).onError(any(Status.class)); // Resets backoff and retry immediately inOrder.verify(backoffPolicyProvider).get(); @@ -2886,9 +2886,9 @@ public void streamClosedAndRetry() { responseObserver.onError(Status.UNAVAILABLE.asException()); verify(configWatcher, times(5)).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); - verify(clusterWatcher, times(5)).onError(statusCaptor.capture()); + verify(cdsResourceWatcher, times(5)).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); - verify(endpointWatcher, times(5)).onError(statusCaptor.capture()); + verify(edsResourceWatcher, times(5)).onError(statusCaptor.capture()); assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); inOrder.verify(backoffPolicy2).nextBackoffNanos(); assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); @@ -2955,7 +2955,7 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); // Start watching cluster information while RPC stream is still in retry backoff. - xdsClient.watchClusterData("cluster.googleapis.com", clusterWatcher); + xdsClient.watchCdsResource("cluster.googleapis.com", cdsResourceWatcher); // Retry after backoff. fakeClock.forwardNanos(9L); @@ -2979,7 +2979,7 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); // Start watching endpoint information while RPC stream is still in retry backoff. - xdsClient.watchEndpointData("cluster.googleapis.com", endpointWatcher); + xdsClient.watchEdsResource("cluster.googleapis.com", edsResourceWatcher); // Retry after backoff. fakeClock.forwardNanos(99L); @@ -3010,14 +3010,14 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { // Client sent an CDS ACK request (Omitted). // No longer interested in endpoint information after RPC resumes. - xdsClient.cancelEndpointDataWatch("cluster.googleapis.com", endpointWatcher); + xdsClient.cancelEdsResourceWatch("cluster.googleapis.com", edsResourceWatcher); // Client updates EDS resource subscription immediately. verify(requestObserver) .onNext(eq(buildDiscoveryRequestV2(NODE, "", ImmutableList.of(), XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); // Become interested in endpoints of another cluster. - xdsClient.watchEndpointData("cluster2.googleapis.com", endpointWatcher); + xdsClient.watchEdsResource("cluster2.googleapis.com", edsResourceWatcher); // Client updates EDS resource subscription immediately. verify(requestObserver) .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster2.googleapis.com", @@ -3049,8 +3049,8 @@ public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); // No longer interested in previous cluster and endpoints in that cluster. - xdsClient.cancelClusterDataWatch("cluster.googleapis.com", clusterWatcher); - xdsClient.cancelEndpointDataWatch("cluster2.googleapis.com", endpointWatcher); + xdsClient.cancelCdsResourceWatch("cluster.googleapis.com", cdsResourceWatcher); + xdsClient.cancelEdsResourceWatch("cluster2.googleapis.com", edsResourceWatcher); // Retry after backoff. fakeClock.forwardNanos(19L); @@ -3174,7 +3174,7 @@ public void streamClosedAndRetryReschedulesAllResourceFetchTimer() { // Client/server resumed LDS/RDS request/response (Omitted). // Start watching cluster data. - xdsClient.watchClusterData("cluster-foo.googleapis.com", clusterWatcher); + xdsClient.watchCdsResource("cluster-foo.googleapis.com", cdsResourceWatcher); ScheduledTask cdsRespTimeoutTask = Iterables.getOnlyElement( fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); @@ -3202,7 +3202,7 @@ public void streamClosedAndRetryReschedulesAllResourceFetchTimer() { .isEqualTo(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC); // Start watching endpoint data. - xdsClient.watchEndpointData("cluster-foo.googleapis.com", endpointWatcher); + xdsClient.watchEdsResource("cluster-foo.googleapis.com", edsResourceWatcher); ScheduledTask edsTimeoutTask = Iterables.getOnlyElement( fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); From 10b960ea5dc3e7d14574c82c275a1dd0904ef3fe Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Thu, 24 Sep 2020 15:08:05 -0700 Subject: [PATCH 67/86] xds: shut down EDS downstream LB policies when no usable endpoints received (#7452) --- .../java/io/grpc/xds/EdsLoadBalancer2.java | 25 +++++++++---------- .../io/grpc/xds/EdsLoadBalancer2Test.java | 21 ++++++++++++++++ 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/EdsLoadBalancer2.java b/xds/src/main/java/io/grpc/xds/EdsLoadBalancer2.java index ece7baf742c..4ba768030ab 100644 --- a/xds/src/main/java/io/grpc/xds/EdsLoadBalancer2.java +++ b/xds/src/main/java/io/grpc/xds/EdsLoadBalancer2.java @@ -275,10 +275,8 @@ public void onChanged(EdsUpdate update) { locality, localityLbInfo.getLocalityWeight()); } if (prioritizedLocalityWeights.isEmpty()) { - lbHelper.helper.updateBalancingState( - TRANSIENT_FAILURE, - new ErrorPicker( - Status.UNAVAILABLE.withDescription("No usable priority/locality/endpoint"))); + propagateResourceError( + Status.UNAVAILABLE.withDescription("No usable priority/locality/endpoint")); return; } if (lb == null) { @@ -310,15 +308,8 @@ public void onChanged(EdsUpdate update) { @Override public void onResourceDoesNotExist(String resourceName) { logger.log(XdsLogLevel.INFO, "Resource {0} is unavailable", resourceName); - if (lb != null) { - lb.shutdown(); - lb = null; - } - lbHelper.helper.updateBalancingState( - TRANSIENT_FAILURE, - new ErrorPicker( - Status.UNAVAILABLE.withDescription( - "Resource " + resourceName + " is unavailable"))); + propagateResourceError( + Status.UNAVAILABLE.withDescription("Resource " + resourceName + " is unavailable")); } @Override @@ -330,6 +321,14 @@ public void onError(Status error) { } } + private void propagateResourceError(Status error) { + if (lb != null) { + lb.shutdown(); + lb = null; + } + lbHelper.helper.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(error)); + } + private final class DropHandlingLbHelper extends ForwardingLoadBalancerHelper { private final Helper helper; private List dropPolicies = Collections.emptyList(); diff --git a/xds/src/test/java/io/grpc/xds/EdsLoadBalancer2Test.java b/xds/src/test/java/io/grpc/xds/EdsLoadBalancer2Test.java index ecb83c94607..8ceb5f7ded8 100644 --- a/xds/src/test/java/io/grpc/xds/EdsLoadBalancer2Test.java +++ b/xds/src/test/java/io/grpc/xds/EdsLoadBalancer2Test.java @@ -406,6 +406,27 @@ public void handleEndpointResource_errorIfNoUsableEndpoints() { .isEqualTo("No usable priority/locality/endpoint"); } + @Test + public void handleEndpointResource_shutDownExistingChildLbPoliciesIfNoUsableEndpoints() { + deliverSimpleClusterLoadAssignment(EDS_SERVICE_NAME); + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(downstreamBalancers); + assertThat(childBalancer.shutdown).isFalse(); + + EquivalentAddressGroup endpoint1 = makeAddress("endpoint-addr-1"); + LocalityLbEndpoints localityLbEndpoints1 = + buildLocalityLbEndpoints(1, 10, Collections.singletonMap(endpoint1, false)); + xdsClient.deliverClusterLoadAssignment( + EDS_SERVICE_NAME, Collections.singletonMap(locality1, localityLbEndpoints1)); + + assertThat(childBalancer.shutdown).isTrue(); + assertThat(currentState).isEqualTo(ConnectivityState.TRANSIENT_FAILURE); + PickResult result = currentPicker.pickSubchannel(mock(PickSubchannelArgs.class)); + assertThat(result.getStatus().isOk()).isFalse(); + assertThat(result.getStatus().getCode()).isEqualTo(Code.UNAVAILABLE); + assertThat(result.getStatus().getDescription()) + .isEqualTo("No usable priority/locality/endpoint"); + } + @Test public void handleDrops() { FakeLoadBalancerProvider fakeRoundRobinProvider = new FakeLoadBalancerProvider("round_robin"); From 7ca6c02312dfe24981034fc4440662eaa3e9613c Mon Sep 17 00:00:00 2001 From: Ran Date: Fri, 25 Sep 2020 10:38:21 -0700 Subject: [PATCH 68/86] Revert "core: delay sending cancel request on client-side when deadline expires (#6328)" (#7457) --- .../java/io/grpc/internal/ClientCallImpl.java | 168 +++++++----------- .../io/grpc/internal/ClientCallImplTest.java | 36 +--- .../grpc/internal/ManagedChannelImplTest.java | 5 +- 3 files changed, 76 insertions(+), 133 deletions(-) diff --git a/core/src/main/java/io/grpc/internal/ClientCallImpl.java b/core/src/main/java/io/grpc/internal/ClientCallImpl.java index db01fad6b3a..3c3a2227780 100644 --- a/core/src/main/java/io/grpc/internal/ClientCallImpl.java +++ b/core/src/main/java/io/grpc/internal/ClientCallImpl.java @@ -71,13 +71,6 @@ final class ClientCallImpl extends ClientCall { private static final Logger log = Logger.getLogger(ClientCallImpl.class.getName()); private static final byte[] FULL_STREAM_DECOMPRESSION_ENCODINGS = "gzip".getBytes(Charset.forName("US-ASCII")); - // When a deadline is exceeded, there is a race between the server receiving the cancellation from - // the client and the server cancelling the stream itself. If the client's cancellation is - // received first, then the stream's status will be CANCELLED instead of DEADLINE_EXCEEDED. - // This prevents server monitoring from noticing high rate of DEADLINE_EXCEEDED, a common - // monitoring metric (b/118879795). Mitigate this by delayed sending of the client's cancellation. - @VisibleForTesting - static final long DEADLINE_EXPIRATION_CANCEL_DELAY_NANOS = TimeUnit.SECONDS.toNanos(1); private final MethodDescriptor method; private final Tag tag; @@ -85,6 +78,7 @@ final class ClientCallImpl extends ClientCall { private final boolean callExecutorIsDirect; private final CallTracer channelCallsTracer; private final Context context; + private volatile ScheduledFuture deadlineCancellationFuture; private final boolean unaryRequest; private CallOptions callOptions; private ClientStream stream; @@ -92,16 +86,14 @@ final class ClientCallImpl extends ClientCall { private boolean cancelCalled; private boolean halfCloseCalled; private final ClientStreamProvider clientStreamProvider; - private ContextCancellationListener cancellationListener; + private final ContextCancellationListener cancellationListener = + new ContextCancellationListener(); private final ScheduledExecutorService deadlineCancellationExecutor; @Nullable private final InternalConfigSelector configSelector; private boolean fullStreamDecompression; private DecompressorRegistry decompressorRegistry = DecompressorRegistry.getDefaultInstance(); private CompressorRegistry compressorRegistry = CompressorRegistry.getDefaultInstance(); - private volatile ScheduledFuture deadlineCancellationNotifyApplicationFuture; - private volatile ScheduledFuture deadlineCancellationSendToServerFuture; - private boolean observerClosed = false; ClientCallImpl( MethodDescriptor method, Executor executor, CallOptions callOptions, @@ -135,20 +127,9 @@ final class ClientCallImpl extends ClientCall { } private final class ContextCancellationListener implements CancellationListener { - private Listener observer; - - private ContextCancellationListener(Listener observer) { - this.observer = observer; - } - @Override public void cancelled(Context context) { - if (context.getDeadline() == null || !context.getDeadline().isExpired()) { - stream.cancel(statusFromCancelled(context)); - } else { - Status status = statusFromCancelled(context); - delayedCancelOnDeadlineExceeded(status, observer); - } + stream.cancel(statusFromCancelled(context)); } } @@ -223,7 +204,19 @@ private void startInternal(Listener observer, Metadata headers) { // Context is already cancelled so no need to create a real stream, just notify the observer // of cancellation via callback on the executor stream = NoopClientStream.INSTANCE; - executeCloseObserverInContext(observer, statusFromCancelled(context)); + final Listener finalObserver = observer; + class ClosedByContext extends ContextRunnable { + ClosedByContext() { + super(context); + } + + @Override + public void runInContext() { + closeObserver(finalObserver, statusFromCancelled(context), new Metadata()); + } + } + + callExecutor.execute(new ClosedByContext()); return; } @@ -251,9 +244,23 @@ private void startInternal(Listener observer, Metadata headers) { compressor = compressorRegistry.lookupCompressor(compressorName); if (compressor == null) { stream = NoopClientStream.INSTANCE; - Status status = Status.INTERNAL.withDescription( - String.format("Unable to find compressor by name %s", compressorName)); - executeCloseObserverInContext(observer, status); + final Listener finalObserver = observer; + class ClosedByNotFoundCompressor extends ContextRunnable { + ClosedByNotFoundCompressor() { + super(context); + } + + @Override + public void runInContext() { + closeObserver( + finalObserver, + Status.INTERNAL.withDescription( + String.format("Unable to find compressor by name %s", compressorName)), + new Metadata()); + } + } + + callExecutor.execute(new ClosedByNotFoundCompressor()); return; } } else { @@ -294,7 +301,6 @@ private void startInternal(Listener observer, Metadata headers) { } stream.setDecompressorRegistry(decompressorRegistry); channelCallsTracer.reportCallStarted(); - cancellationListener = new ContextCancellationListener(observer); stream.start(new ClientStreamListenerImpl(observer)); // Delay any sources of cancellation after start(), because most of the transports are broken if @@ -306,11 +312,8 @@ private void startInternal(Listener observer, Metadata headers) { // If the context has the effective deadline, we don't need to schedule an extra task. && !effectiveDeadline.equals(context.getDeadline()) // If the channel has been terminated, we don't need to schedule an extra task. - && deadlineCancellationExecutor != null - // if already expired deadline let failing stream handle - && !(stream instanceof FailingClientStream)) { - deadlineCancellationNotifyApplicationFuture = - startDeadlineNotifyApplicationTimer(effectiveDeadline, observer); + && deadlineCancellationExecutor != null) { + deadlineCancellationFuture = startDeadlineTimer(effectiveDeadline); } if (cancelListenersShouldBeRemoved) { // Race detected! ClientStreamListener.closed may have been called before @@ -410,76 +413,46 @@ private static void logIfContextNarrowedTimeout( private void removeContextListenerAndCancelDeadlineFuture() { context.removeListener(cancellationListener); - ScheduledFuture f = deadlineCancellationSendToServerFuture; - if (f != null) { - f.cancel(false); - } - - f = deadlineCancellationNotifyApplicationFuture; + ScheduledFuture f = deadlineCancellationFuture; if (f != null) { f.cancel(false); } } - private ScheduledFuture startDeadlineNotifyApplicationTimer(Deadline deadline, - final Listener observer) { - final long remainingNanos = deadline.timeRemaining(TimeUnit.NANOSECONDS); - - class DeadlineExceededNotifyApplicationTimer implements Runnable { - @Override - public void run() { - Status status = buildDeadlineExceededStatusWithRemainingNanos(remainingNanos); - delayedCancelOnDeadlineExceeded(status, observer); - } - } - - return deadlineCancellationExecutor.schedule( - new LogExceptionRunnable(new DeadlineExceededNotifyApplicationTimer()), - remainingNanos, - TimeUnit.NANOSECONDS); - } - - private Status buildDeadlineExceededStatusWithRemainingNanos(long remainingNanos) { - final InsightBuilder insight = new InsightBuilder(); - stream.appendTimeoutInsight(insight); + private class DeadlineTimer implements Runnable { + private final long remainingNanos; - long seconds = Math.abs(remainingNanos) / TimeUnit.SECONDS.toNanos(1); - long nanos = Math.abs(remainingNanos) % TimeUnit.SECONDS.toNanos(1); - - StringBuilder buf = new StringBuilder(); - buf.append("deadline exceeded after "); - if (remainingNanos < 0) { - buf.append('-'); - } - buf.append(seconds); - buf.append(String.format(".%09d", nanos)); - buf.append("s. "); - buf.append(insight); - - return DEADLINE_EXCEEDED.augmentDescription(buf.toString()); - } - - private void delayedCancelOnDeadlineExceeded(final Status status, Listener observer) { - if (deadlineCancellationSendToServerFuture != null) { - return; + DeadlineTimer(long remainingNanos) { + this.remainingNanos = remainingNanos; } - class DeadlineExceededSendCancelToServerTimer implements Runnable { - @Override - public void run() { - // DelayedStream.cancel() is safe to call from a thread that is different from where the - // stream is created. - stream.cancel(status); + @Override + public void run() { + InsightBuilder insight = new InsightBuilder(); + stream.appendTimeoutInsight(insight); + // DelayedStream.cancel() is safe to call from a thread that is different from where the + // stream is created. + long seconds = Math.abs(remainingNanos) / TimeUnit.SECONDS.toNanos(1); + long nanos = Math.abs(remainingNanos) % TimeUnit.SECONDS.toNanos(1); + + StringBuilder buf = new StringBuilder(); + buf.append("deadline exceeded after "); + if (remainingNanos < 0) { + buf.append('-'); } + buf.append(seconds); + buf.append(String.format(".%09d", nanos)); + buf.append("s. "); + buf.append(insight); + stream.cancel(DEADLINE_EXCEEDED.augmentDescription(buf.toString())); } + } - // This races with removeContextListenerAndCancelDeadlineFuture(). Since calling cancel() on a - // stream multiple time is safe, the race here is fine. - deadlineCancellationSendToServerFuture = deadlineCancellationExecutor.schedule( - new LogExceptionRunnable(new DeadlineExceededSendCancelToServerTimer()), - DEADLINE_EXPIRATION_CANCEL_DELAY_NANOS, - TimeUnit.NANOSECONDS); - executeCloseObserverInContext(observer, status); + private ScheduledFuture startDeadlineTimer(Deadline deadline) { + long remainingNanos = deadline.timeRemaining(TimeUnit.NANOSECONDS); + return deadlineCancellationExecutor.schedule( + new LogExceptionRunnable( + new DeadlineTimer(remainingNanos)), remainingNanos, TimeUnit.NANOSECONDS); } private void executeCloseObserverInContext(final Listener observer, final Status status) { @@ -497,13 +470,6 @@ public void runInContext() { callExecutor.execute(new CloseInContext()); } - private void closeObserver(Listener observer, Status status, Metadata trailers) { - if (!observerClosed) { - observerClosed = true; - observer.onClose(status, trailers); - } - } - @Nullable private Deadline effectiveDeadline() { // Call options and context are immutable, so we don't need to cache the deadline. @@ -646,6 +612,10 @@ public Attributes getAttributes() { return Attributes.EMPTY; } + private void closeObserver(Listener observer, Status status, Metadata trailers) { + observer.onClose(status, trailers); + } + @Override public String toString() { return MoreObjects.toStringHelper(this).add("method", method).toString(); diff --git a/core/src/test/java/io/grpc/internal/ClientCallImplTest.java b/core/src/test/java/io/grpc/internal/ClientCallImplTest.java index a84bb225457..4b67454f175 100644 --- a/core/src/test/java/io/grpc/internal/ClientCallImplTest.java +++ b/core/src/test/java/io/grpc/internal/ClientCallImplTest.java @@ -17,7 +17,6 @@ package io.grpc.internal; import static com.google.common.truth.Truth.assertThat; -import static io.grpc.internal.ClientCallImpl.DEADLINE_EXPIRATION_CANCEL_DELAY_NANOS; import static io.grpc.internal.GrpcUtil.ACCEPT_ENCODING_SPLITTER; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; @@ -60,7 +59,6 @@ import io.grpc.MethodDescriptor; import io.grpc.MethodDescriptor.MethodType; import io.grpc.Status; -import io.grpc.Status.Code; import io.grpc.internal.ClientCallImpl.ClientStreamProvider; import io.grpc.internal.ManagedChannelServiceConfig.MethodInfo; import io.grpc.internal.testing.SingleMessageProducer; @@ -137,9 +135,6 @@ public class ClientCallImplTest { @Captor private ArgumentCaptor statusArgumentCaptor; - @Captor - private ArgumentCaptor metadataArgumentCaptor; - private CallOptions baseCallOptions; @Before @@ -1005,21 +1000,9 @@ public void expiredDeadlineCancelsStream_CallOptions() { call.start(callListener, new Metadata()); - fakeClock.forwardTime(1000, TimeUnit.MILLISECONDS); - - // Verify cancel sent to application when deadline just past - verify(callListener).onClose(statusCaptor.capture(), metadataArgumentCaptor.capture()); - assertThat(statusCaptor.getValue().getDescription()) - .matches("deadline exceeded after [0-9]+\\.[0-9]+s. \\[remote_addr=127\\.0\\.0\\.1:443\\]"); - assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.DEADLINE_EXCEEDED); - verify(stream, never()).cancel(statusCaptor.capture()); - - fakeClock.forwardNanos(DEADLINE_EXPIRATION_CANCEL_DELAY_NANOS - 1); - verify(stream, never()).cancel(any(Status.class)); + fakeClock.forwardNanos(TimeUnit.SECONDS.toNanos(1) + 1); - // verify cancel send to server is delayed with DEADLINE_EXPIRATION_CANCEL_DELAY - fakeClock.forwardNanos(1); - verify(stream).cancel(statusCaptor.capture()); + verify(stream, times(1)).cancel(statusCaptor.capture()); assertEquals(Status.Code.DEADLINE_EXCEEDED, statusCaptor.getValue().getCode()); assertThat(statusCaptor.getValue().getDescription()) .matches("deadline exceeded after [0-9]+\\.[0-9]+s. \\[remote_addr=127\\.0\\.0\\.1:443\\]"); @@ -1029,8 +1012,8 @@ public void expiredDeadlineCancelsStream_CallOptions() { public void expiredDeadlineCancelsStream_Context() { fakeClock.forwardTime(System.nanoTime(), TimeUnit.NANOSECONDS); - Deadline deadline = Deadline.after(1, TimeUnit.SECONDS, fakeClock.getDeadlineTicker()); - Context context = Context.current().withDeadline(deadline, deadlineCancellationExecutor); + Context context = Context.current() + .withDeadlineAfter(1, TimeUnit.SECONDS, deadlineCancellationExecutor); Context origContext = context.attach(); ClientCallImpl call = new ClientCallImpl<>( @@ -1045,16 +1028,9 @@ public void expiredDeadlineCancelsStream_Context() { call.start(callListener, new Metadata()); - fakeClock.forwardTime(1000, TimeUnit.MILLISECONDS); - verify(stream, never()).cancel(statusCaptor.capture()); - // verify app is notified. - verify(callListener).onClose(statusCaptor.capture(), metadataArgumentCaptor.capture()); - assertThat(statusCaptor.getValue().getDescription()).contains("context timed out"); - assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.DEADLINE_EXCEEDED); + fakeClock.forwardNanos(TimeUnit.SECONDS.toNanos(1) + 1); - // verify cancel send to server is delayed with DEADLINE_EXPIRATION_CANCEL_DELAY - fakeClock.forwardNanos(DEADLINE_EXPIRATION_CANCEL_DELAY_NANOS); - verify(stream).cancel(statusCaptor.capture()); + verify(stream, times(1)).cancel(statusCaptor.capture()); assertEquals(Status.Code.DEADLINE_EXCEEDED, statusCaptor.getValue().getCode()); assertThat(statusCaptor.getValue().getDescription()).isEqualTo("context timed out"); } diff --git a/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java b/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java index 65a9ed31e29..74097162618 100644 --- a/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java +++ b/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java @@ -3622,7 +3622,7 @@ public ClientTransportFactory buildClientTransportFactory() { CallOptions.DEFAULT.withDeadlineAfter(5, TimeUnit.SECONDS)); ListenableFuture future2 = ClientCalls.futureUnaryCall(call2, null); - timer.forwardTime(5, TimeUnit.SECONDS); + timer.forwardTime(1234, TimeUnit.SECONDS); executor.runDueTasks(); try { @@ -3633,9 +3633,6 @@ public ClientTransportFactory buildClientTransportFactory() { } mychannel.shutdownNow(); - // Now for Deadline_exceeded, stream shutdown is delayed, calling shutdownNow() on a open stream - // will add a task to executor. Cleaning that task here. - executor.runDueTasks(); } @Deprecated From 9cbea16ccc5b47c5323754dcb0a4f77c0e2209ad Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Fri, 25 Sep 2020 17:50:25 -0700 Subject: [PATCH 69/86] xds: stop setting PROXYLESS_CLIENT_HOSTNAME node metadata in LRS requests (#7459) The PROXYLESS_CLIENT_HOSTNAME node metadata was a temporary workaround for management server to not send back all backend services as load reporting clusters. Now the management server is able to use `send_all_clusters` field to let the client side decide the group of clusters it is reporting loads for. So this node metadata is no longer needed. --- .../main/java/io/grpc/xds/LoadReportClient.java | 14 +------------- .../java/io/grpc/xds/LoadReportClientTest.java | 10 +--------- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/LoadReportClient.java b/xds/src/main/java/io/grpc/xds/LoadReportClient.java index 48d7b0a5cb1..d39db86ca16 100644 --- a/xds/src/main/java/io/grpc/xds/LoadReportClient.java +++ b/xds/src/main/java/io/grpc/xds/LoadReportClient.java @@ -39,9 +39,7 @@ import io.grpc.xds.XdsClient.XdsChannel; import io.grpc.xds.XdsLogger.XdsLogLevel; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; @@ -53,9 +51,6 @@ */ @NotThreadSafe final class LoadReportClient { - @VisibleForTesting - static final String TARGET_NAME_METADATA_KEY = "PROXYLESS_CLIENT_HOSTNAME"; - private final InternalLogId logId; private final XdsLogger logger; private final XdsChannel xdsChannel; @@ -91,14 +86,7 @@ final class LoadReportClient { this.backoffPolicyProvider = checkNotNull(backoffPolicyProvider, "backoffPolicyProvider"); checkNotNull(stopwatchSupplier, "stopwatchSupplier"); this.retryStopwatch = stopwatchSupplier.get(); - checkNotNull(targetName, "targetName"); - checkNotNull(node, "node"); - Map newMetadata = new HashMap<>(); - if (node.getMetadata() != null) { - newMetadata.putAll(node.getMetadata()); - } - newMetadata.put(TARGET_NAME_METADATA_KEY, targetName); - this.node = node.toBuilder().setMetadata(newMetadata).build(); + this.node = checkNotNull(node, "node"); logId = InternalLogId.allocate("lrs-client", targetName); logger = XdsLogger.withLogId(logId); logger.log(XdsLogLevel.INFO, "Created"); diff --git a/xds/src/test/java/io/grpc/xds/LoadReportClientTest.java b/xds/src/test/java/io/grpc/xds/LoadReportClientTest.java index 0a2951ad4dc..74daf52693d 100644 --- a/xds/src/test/java/io/grpc/xds/LoadReportClientTest.java +++ b/xds/src/test/java/io/grpc/xds/LoadReportClientTest.java @@ -469,10 +469,7 @@ private static LoadStatsRequest buildInitialRequest() { Struct.newBuilder() .putFields( "TRAFFICDIRECTOR_NETWORK_HOSTNAME", - Value.newBuilder().setStringValue("default").build()) - .putFields( - LoadReportClient.TARGET_NAME_METADATA_KEY, - Value.newBuilder().setStringValue(TARGET_NAME).build()))) + Value.newBuilder().setStringValue("default").build()))) .build(); } @@ -490,11 +487,6 @@ private static class LoadStatsRequestMatcher implements ArgumentMatcher Date: Mon, 28 Sep 2020 09:56:01 -0700 Subject: [PATCH 70/86] rls: fix RLS_DATA_KEY propagation in headers --- .../java/io/grpc/rls/CachingRlsLbClient.java | 27 +++++++------------ .../io/grpc/rls/CachingRlsLbClientTest.java | 17 +++++++++--- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/rls/src/main/java/io/grpc/rls/CachingRlsLbClient.java b/rls/src/main/java/io/grpc/rls/CachingRlsLbClient.java index 4e0acb9392a..e1a4152d896 100644 --- a/rls/src/main/java/io/grpc/rls/CachingRlsLbClient.java +++ b/rls/src/main/java/io/grpc/rls/CachingRlsLbClient.java @@ -43,7 +43,6 @@ import io.grpc.SynchronizationContext.ScheduledHandle; import io.grpc.internal.BackoffPolicy; import io.grpc.internal.ExponentialBackoffPolicy; -import io.grpc.internal.PickSubchannelArgsImpl; import io.grpc.internal.TimeProvider; import io.grpc.lookup.v1.RouteLookupServiceGrpc; import io.grpc.lookup.v1.RouteLookupServiceGrpc.RouteLookupServiceStub; @@ -204,7 +203,6 @@ public void onCompleted() { */ @CheckReturnValue final CachedRouteLookupResponse get(final RouteLookupRequest request) { - synchronizationContext.throwIfNotInThisSynchronizationContext(); synchronized (lock) { final CacheEntry cacheEntry; cacheEntry = linkedHashLruCache.read(request); @@ -844,6 +842,7 @@ public void onStatusChanged(ConnectivityState newState) { } /** A header will be added when RLS server respond with additional header data. */ + @VisibleForTesting static final Metadata.Key RLS_DATA_KEY = Metadata.Key.of("X-Google-RLS-Data", Metadata.ASCII_STRING_MARSHALLER); @@ -862,7 +861,11 @@ public PickResult pickSubchannel(PickSubchannelArgs args) { requestFactory.create(methodName[0], methodName[1], args.getHeaders()); final CachedRouteLookupResponse response = CachingRlsLbClient.this.get(request); - PickSubchannelArgs rlsAppliedArgs = getApplyRlsHeader(args, response); + if (response.getHeaderData() != null && !response.getHeaderData().isEmpty()) { + Metadata headers = args.getHeaders(); + headers.discardAll(RLS_DATA_KEY); + headers.put(RLS_DATA_KEY, response.getHeaderData()); + } if (response.hasData()) { ChildPolicyWrapper childPolicyWrapper = response.getChildPolicyWrapper(); ConnectivityState connectivityState = @@ -872,31 +875,19 @@ public PickResult pickSubchannel(PickSubchannelArgs args) { case CONNECTING: return PickResult.withNoResult(); case READY: - return childPolicyWrapper.getPicker().pickSubchannel(rlsAppliedArgs); + return childPolicyWrapper.getPicker().pickSubchannel(args); case TRANSIENT_FAILURE: case SHUTDOWN: default: - return useFallback(rlsAppliedArgs); + return useFallback(args); } } else if (response.hasError()) { - return useFallback(rlsAppliedArgs); + return useFallback(args); } else { return PickResult.withNoResult(); } } - private PickSubchannelArgs getApplyRlsHeader( - PickSubchannelArgs args, CachedRouteLookupResponse response) { - if (response.getHeaderData() == null || response.getHeaderData().isEmpty()) { - return args; - } - - Metadata headers = new Metadata(); - headers.merge(args.getHeaders()); - headers.put(RLS_DATA_KEY, response.getHeaderData()); - return new PickSubchannelArgsImpl(args.getMethodDescriptor(), headers, args.getCallOptions()); - } - private ChildPolicyWrapper fallbackChildPolicyWrapper; /** Uses Subchannel connected to default target. */ diff --git a/rls/src/test/java/io/grpc/rls/CachingRlsLbClientTest.java b/rls/src/test/java/io/grpc/rls/CachingRlsLbClientTest.java index c58ff3e210a..8d7a4febbaa 100644 --- a/rls/src/test/java/io/grpc/rls/CachingRlsLbClientTest.java +++ b/rls/src/test/java/io/grpc/rls/CachingRlsLbClientTest.java @@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.truth.Truth.assertThat; +import static io.grpc.rls.CachingRlsLbClient.RLS_DATA_KEY; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.CALLS_REAL_METHODS; @@ -33,6 +34,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.SettableFuture; import io.grpc.Attributes; +import io.grpc.CallOptions; import io.grpc.ConnectivityState; import io.grpc.EquivalentAddressGroup; import io.grpc.ForwardingChannelBuilder; @@ -42,12 +44,14 @@ import io.grpc.LoadBalancerProvider; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; +import io.grpc.Metadata; import io.grpc.NameResolver; import io.grpc.Status; import io.grpc.SynchronizationContext; import io.grpc.inprocess.InProcessChannelBuilder; import io.grpc.inprocess.InProcessServerBuilder; import io.grpc.internal.BackoffPolicy; +import io.grpc.internal.PickSubchannelArgsImpl; import io.grpc.lookup.v1.RouteLookupServiceGrpc; import io.grpc.rls.CachingRlsLbClient.CacheEntry; import io.grpc.rls.CachingRlsLbClient.CachedRouteLookupResponse; @@ -66,6 +70,7 @@ import io.grpc.rls.RlsProtoData.RouteLookupResponse; import io.grpc.stub.StreamObserver; import io.grpc.testing.GrpcCleanupRule; +import io.grpc.testing.TestMethodDescriptors; import java.io.IOException; import java.lang.Thread.UncaughtExceptionHandler; import java.net.SocketAddress; @@ -268,11 +273,11 @@ public void get_throttledAndRecover() throws Exception { public void get_updatesLbState() throws Exception { InOrder inOrder = inOrder(helper); RouteLookupRequest routeLookupRequest = - new RouteLookupRequest("server", "/foo/bar", "grpc", ImmutableMap.of()); + new RouteLookupRequest("service1", "/foo/bar", "grpc", ImmutableMap.of()); rlsServerImpl.setLookupTable( ImmutableMap.of( routeLookupRequest, - new RouteLookupResponse(ImmutableList.of("target"), "header"))); + new RouteLookupResponse(ImmutableList.of("target"), "header-rls-data-value"))); // valid channel CachedRouteLookupResponse resp = getInSyncContext(routeLookupRequest); @@ -291,7 +296,13 @@ public void get_updatesLbState() throws Exception { assertThat(new HashSet<>(pickerCaptor.getAllValues())).hasSize(1); assertThat(stateCaptor.getAllValues()) .containsExactly(ConnectivityState.CONNECTING, ConnectivityState.READY); - assertThat(pickerCaptor.getValue()).isInstanceOf(RlsPicker.class); + Metadata headers = new Metadata(); + pickerCaptor.getValue().pickSubchannel( + new PickSubchannelArgsImpl( + TestMethodDescriptors.voidMethod().toBuilder().setFullMethodName("foo/bar").build(), + headers, + CallOptions.DEFAULT)); + assertThat(headers.get(RLS_DATA_KEY)).isEqualTo("header-rls-data-value"); // move backoff further back to only test error behavior fakeBackoffProvider.nextPolicy = createBackoffPolicy(100, TimeUnit.MILLISECONDS); From 950ec30247ea5a0e992060c00ccac37aa8948279 Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Mon, 28 Sep 2020 09:59:39 -0700 Subject: [PATCH 71/86] xds: delete XdsClientImplV2Test (#7461) Maintaining two copies of tests is counter-productive. Having the entire set of XdsClientImpl tests for covering v2 protocol usage is an overkill. --- .../java/io/grpc/xds/XdsClientImplTestV2.java | 3753 ----------------- 1 file changed, 3753 deletions(-) delete mode 100644 xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java deleted file mode 100644 index e04cd739aba..00000000000 --- a/xds/src/test/java/io/grpc/xds/XdsClientImplTestV2.java +++ /dev/null @@ -1,3753 +0,0 @@ -/* - * Copyright 2019 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.xds; - -import static com.google.common.truth.Truth.assertThat; -import static io.grpc.xds.XdsClientTestHelper.buildClusterLoadAssignmentV2; -import static io.grpc.xds.XdsClientTestHelper.buildClusterV2; -import static io.grpc.xds.XdsClientTestHelper.buildDiscoveryRequestV2; -import static io.grpc.xds.XdsClientTestHelper.buildDiscoveryResponseV2; -import static io.grpc.xds.XdsClientTestHelper.buildDropOverloadV2; -import static io.grpc.xds.XdsClientTestHelper.buildLbEndpointV2; -import static io.grpc.xds.XdsClientTestHelper.buildListenerV2; -import static io.grpc.xds.XdsClientTestHelper.buildLocalityLbEndpointsV2; -import static io.grpc.xds.XdsClientTestHelper.buildRouteConfigurationV2; -import static io.grpc.xds.XdsClientTestHelper.buildSecureClusterV2; -import static io.grpc.xds.XdsClientTestHelper.buildUpstreamTlsContextV2; -import static io.grpc.xds.XdsClientTestHelper.buildVirtualHostV2; -import static org.mockito.AdditionalAnswers.delegatesTo; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.argThat; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.protobuf.Any; -import com.google.protobuf.BoolValue; -import com.google.protobuf.UInt32Value; -import com.google.protobuf.util.Durations; -import io.envoyproxy.envoy.api.v2.ClusterLoadAssignment.Policy; -import io.envoyproxy.envoy.api.v2.DiscoveryRequest; -import io.envoyproxy.envoy.api.v2.DiscoveryResponse; -import io.envoyproxy.envoy.api.v2.auth.UpstreamTlsContext; -import io.envoyproxy.envoy.api.v2.core.AggregatedConfigSource; -import io.envoyproxy.envoy.api.v2.core.ConfigSource; -import io.envoyproxy.envoy.api.v2.core.HealthStatus; -import io.envoyproxy.envoy.api.v2.endpoint.ClusterStats; -import io.envoyproxy.envoy.api.v2.route.RedirectAction; -import io.envoyproxy.envoy.api.v2.route.WeightedCluster; -import io.envoyproxy.envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager; -import io.envoyproxy.envoy.config.filter.network.http_connection_manager.v2.Rds; -import io.envoyproxy.envoy.config.route.v3.QueryParameterMatcher; -import io.envoyproxy.envoy.config.route.v3.Route; -import io.envoyproxy.envoy.config.route.v3.RouteAction; -import io.envoyproxy.envoy.config.route.v3.RouteConfiguration; -import io.envoyproxy.envoy.config.route.v3.RouteMatch; -import io.envoyproxy.envoy.config.route.v3.VirtualHost; -import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.SdsSecretConfig; -import io.envoyproxy.envoy.service.discovery.v2.AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceImplBase; -import io.envoyproxy.envoy.service.load_stats.v2.LoadReportingServiceGrpc.LoadReportingServiceImplBase; -import io.envoyproxy.envoy.service.load_stats.v2.LoadStatsRequest; -import io.envoyproxy.envoy.service.load_stats.v2.LoadStatsResponse; -import io.grpc.Context; -import io.grpc.Context.CancellationListener; -import io.grpc.ManagedChannel; -import io.grpc.Status; -import io.grpc.Status.Code; -import io.grpc.SynchronizationContext; -import io.grpc.inprocess.InProcessChannelBuilder; -import io.grpc.inprocess.InProcessServerBuilder; -import io.grpc.internal.BackoffPolicy; -import io.grpc.internal.FakeClock; -import io.grpc.internal.FakeClock.ScheduledTask; -import io.grpc.internal.FakeClock.TaskFilter; -import io.grpc.stub.StreamObserver; -import io.grpc.testing.GrpcCleanupRule; -import io.grpc.xds.EnvoyProtoData.DropOverload; -import io.grpc.xds.EnvoyProtoData.LbEndpoint; -import io.grpc.xds.EnvoyProtoData.Locality; -import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints; -import io.grpc.xds.EnvoyProtoData.Node; -import io.grpc.xds.XdsClient.CdsResourceWatcher; -import io.grpc.xds.XdsClient.CdsUpdate; -import io.grpc.xds.XdsClient.ConfigUpdate; -import io.grpc.xds.XdsClient.ConfigWatcher; -import io.grpc.xds.XdsClient.EdsResourceWatcher; -import io.grpc.xds.XdsClient.EdsUpdate; -import io.grpc.xds.XdsClient.XdsChannel; -import io.grpc.xds.XdsClientImpl.MessagePrinter; -import java.io.IOException; -import java.util.ArrayDeque; -import java.util.HashSet; -import java.util.List; -import java.util.Queue; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.mockito.ArgumentCaptor; -import org.mockito.ArgumentMatcher; -import org.mockito.InOrder; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; - -/** - * Tests for {@link XdsClientImpl} with xDS v2 protocol. - */ -@RunWith(JUnit4.class) -public class XdsClientImplTestV2 { - - private static final String TARGET_AUTHORITY = "foo.googleapis.com:8080"; - - private static final Node NODE = Node.newBuilder().build(); - private static final TaskFilter RPC_RETRY_TASK_FILTER = - new TaskFilter() { - @Override - public boolean shouldAccept(Runnable command) { - return command.toString().contains(XdsClientImpl.RpcRetryTask.class.getSimpleName()); - } - }; - - private static final TaskFilter LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER = - new TaskFilter() { - @Override - public boolean shouldAccept(Runnable command) { - return command.toString() - .contains(XdsClientImpl.LdsResourceFetchTimeoutTask.class.getSimpleName()); - } - }; - - private static final TaskFilter RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER = - new TaskFilter() { - @Override - public boolean shouldAccept(Runnable command) { - return command.toString() - .contains(XdsClientImpl.RdsResourceFetchTimeoutTask.class.getSimpleName()); - } - }; - - private static final TaskFilter CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER = - new TaskFilter() { - @Override - public boolean shouldAccept(Runnable command) { - return command.toString() - .contains(XdsClientImpl.CdsResourceFetchTimeoutTask.class.getSimpleName()); - } - }; - - private static final TaskFilter EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER = - new TaskFilter() { - @Override - public boolean shouldAccept(Runnable command) { - return command.toString() - .contains(XdsClientImpl.EdsResourceFetchTimeoutTask.class.getSimpleName()); - } - }; - - @Rule - public final GrpcCleanupRule cleanupRule = new GrpcCleanupRule(); - @Rule - public ExpectedException thrown = ExpectedException.none(); - - private final SynchronizationContext syncContext = new SynchronizationContext( - new Thread.UncaughtExceptionHandler() { - @Override - public void uncaughtException(Thread t, Throwable e) { - throw new AssertionError(e); - } - }); - private final FakeClock fakeClock = new FakeClock(); - - private final Queue> responseObservers = new ArrayDeque<>(); - private final Queue> requestObservers = new ArrayDeque<>(); - private final AtomicBoolean adsEnded = new AtomicBoolean(true); - private final Queue loadReportCalls = new ArrayDeque<>(); - private final AtomicBoolean lrsEnded = new AtomicBoolean(true); - - @Mock - private AggregatedDiscoveryServiceImplBase mockedDiscoveryService; - @Mock - private BackoffPolicy.Provider backoffPolicyProvider; - @Mock - private BackoffPolicy backoffPolicy1; - @Mock - private BackoffPolicy backoffPolicy2; - @Mock - private ConfigWatcher configWatcher; - @Mock - private CdsResourceWatcher cdsResourceWatcher; - @Mock - private EdsResourceWatcher edsResourceWatcher; - - private ManagedChannel channel; - private XdsClientImpl xdsClient; - - @Before - public void setUp() throws IOException { - MockitoAnnotations.initMocks(this); - when(backoffPolicyProvider.get()).thenReturn(backoffPolicy1, backoffPolicy2); - when(backoffPolicy1.nextBackoffNanos()).thenReturn(10L, 100L); - when(backoffPolicy2.nextBackoffNanos()).thenReturn(20L, 200L); - - final String serverName = InProcessServerBuilder.generateName(); - AggregatedDiscoveryServiceImplBase adsServiceImpl = new AggregatedDiscoveryServiceImplBase() { - @Override - public StreamObserver streamAggregatedResources( - final StreamObserver responseObserver) { - assertThat(adsEnded.get()).isTrue(); // ensure previous call was ended - adsEnded.set(false); - Context.current().addListener( - new CancellationListener() { - @Override - public void cancelled(Context context) { - adsEnded.set(true); - } - }, MoreExecutors.directExecutor()); - responseObservers.offer(responseObserver); - @SuppressWarnings("unchecked") - StreamObserver requestObserver = mock(StreamObserver.class); - requestObservers.offer(requestObserver); - return requestObserver; - } - }; - mockedDiscoveryService = - mock(AggregatedDiscoveryServiceImplBase.class, delegatesTo(adsServiceImpl)); - - LoadReportingServiceImplBase lrsServiceImpl = new LoadReportingServiceImplBase() { - @Override - public StreamObserver streamLoadStats( - StreamObserver responseObserver) { - assertThat(lrsEnded.get()).isTrue(); - lrsEnded.set(false); - @SuppressWarnings("unchecked") - StreamObserver requestObserver = mock(StreamObserver.class); - final LoadReportCall call = new LoadReportCall(requestObserver, responseObserver); - Context.current().addListener( - new CancellationListener() { - @Override - public void cancelled(Context context) { - lrsEnded.set(true); - } - }, MoreExecutors.directExecutor()); - loadReportCalls.offer(call); - return requestObserver; - } - }; - - cleanupRule.register( - InProcessServerBuilder - .forName(serverName) - .addService(mockedDiscoveryService) - .addService(lrsServiceImpl) - .directExecutor() - .build() - .start()); - channel = - cleanupRule.register(InProcessChannelBuilder.forName(serverName).directExecutor().build()); - - xdsClient = - new XdsClientImpl( - TARGET_AUTHORITY, - new XdsChannel(channel, /* useProtocolV3= */ false), - Node.newBuilder().build(), - syncContext, - fakeClock.getScheduledExecutorService(), - backoffPolicyProvider, - fakeClock.getStopwatchSupplier()); - // Only the connection to management server is established, no RPC request is sent until at - // least one watcher is registered. - assertThat(responseObservers).isEmpty(); - assertThat(requestObservers).isEmpty(); - } - - @After - public void tearDown() { - xdsClient.shutdown(); - assertThat(adsEnded.get()).isTrue(); - assertThat(lrsEnded.get()).isTrue(); - assertThat(channel.isShutdown()).isTrue(); - assertThat(fakeClock.getPendingTasks()).isEmpty(); - } - - // Always test the real workflow and integrity of XdsClient: RDS protocol should always followed - // after at least one LDS request-response, from which the RDS resource name comes. CDS and EDS - // can be tested separately as they are used in a standalone way. - - // Discovery responses should follow management server spec and xDS protocol. See - // https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol. - - /** - * Client receives an LDS response that does not contain a Listener for the requested resource. - * The LDS response is ACKed. - * The config watcher is notified with resource unavailable after its response timer expires. - */ - @Test - public void ldsResponseWithoutMatchingResource() { - xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - // Client sends an LDS request for the host name (with port) to management server. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - - assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); - - List listeners = ImmutableList.of( - Any.pack(buildListenerV2("bar.googleapis.com", - Any.pack(HttpConnectionManager.newBuilder() - .setRouteConfig( - buildRouteConfigurationV2("route-bar.googleapis.com", - ImmutableList.of( - buildVirtualHostV2( - ImmutableList.of("bar.googleapis.com"), - "cluster-bar.googleapis.com")))) - .build()))), - Any.pack(buildListenerV2("baz.googleapis.com", - Any.pack(HttpConnectionManager.newBuilder() - .setRouteConfig( - buildRouteConfigurationV2("route-baz.googleapis.com", - ImmutableList.of( - buildVirtualHostV2( - ImmutableList.of("baz.googleapis.com"), - "cluster-baz.googleapis.com")))) - .build())))); - DiscoveryResponse response = - buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sends an ACK LDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); - - verify(configWatcher, never()).onConfigChanged(any(ConfigUpdate.class)); - verify(configWatcher, never()).onResourceDoesNotExist(TARGET_AUTHORITY); - verify(configWatcher, never()).onError(any(Status.class)); - fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); - verify(configWatcher).onResourceDoesNotExist(TARGET_AUTHORITY); - assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); - } - - /** - * An LDS response contains the requested listener and an in-lined RouteConfiguration message for - * that listener. But the RouteConfiguration message is invalid as it does not contain any - * VirtualHost with domains matching the requested hostname. - * The LDS response is NACKed, as if the XdsClient has not received this response. - * The config watcher is notified with an error after its response timer expires.. - */ - @Test - public void failToFindVirtualHostInLdsResponseInLineRouteConfig() { - xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - // Client sends an LDS request for the host name (with port) to management server. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); - - io.envoyproxy.envoy.api.v2.RouteConfiguration routeConfig = - buildRouteConfigurationV2( - "route.googleapis.com", - ImmutableList.of( - buildVirtualHostV2(ImmutableList.of("something does not match"), - "some cluster"), - buildVirtualHostV2(ImmutableList.of("something else does not match"), - "some other cluster"))); - - List listeners = ImmutableList.of( - Any.pack(buildListenerV2(TARGET_AUTHORITY, /* matching resource */ - Any.pack(HttpConnectionManager.newBuilder().setRouteConfig(routeConfig).build())))); - DiscoveryResponse response = - buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sends an NACK LDS request. - verify(requestObserver) - .onNext( - argThat(new DiscoveryRequestMatcher("", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); - - verify(configWatcher, never()).onConfigChanged(any(ConfigUpdate.class)); - verify(configWatcher, never()).onResourceDoesNotExist(TARGET_AUTHORITY); - verify(configWatcher, never()).onError(any(Status.class)); - - fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); - verify(configWatcher).onResourceDoesNotExist(TARGET_AUTHORITY); - assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); - } - - /** - * Client resolves the virtual host config from an LDS response that contains a - * RouteConfiguration message directly in-line for the requested resource. No RDS is needed. - * The LDS response is ACKed. - * The config watcher is notified with an update. - */ - @Test - public void resolveVirtualHostInLdsResponse() { - xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - // Client sends an LDS request for the host name (with port) to management server. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - ScheduledTask ldsRespTimer = - Iterables.getOnlyElement( - fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); - assertThat(ldsRespTimer.isCancelled()).isFalse(); - - List listeners = ImmutableList.of( - Any.pack(buildListenerV2("bar.googleapis.com", - Any.pack(HttpConnectionManager.newBuilder() - .setRouteConfig( - buildRouteConfigurationV2("route-bar.googleapis.com", - ImmutableList.of( - buildVirtualHostV2( - ImmutableList.of("bar.googleapis.com"), - "cluster-bar.googleapis.com")))) - .build()))), - Any.pack(buildListenerV2("baz.googleapis.com", - Any.pack(HttpConnectionManager.newBuilder() - .setRouteConfig( - buildRouteConfigurationV2("route-baz.googleapis.com", - ImmutableList.of( - buildVirtualHostV2( - ImmutableList.of("baz.googleapis.com"), - "cluster-baz.googleapis.com")))) - .build()))), - Any.pack(buildListenerV2(TARGET_AUTHORITY, /* matching resource */ - Any.pack( - HttpConnectionManager.newBuilder() - .setRouteConfig( // target route configuration - buildRouteConfigurationV2("route-foo.googleapis.com", - ImmutableList.of( - buildVirtualHostV2( // matching virtual host - ImmutableList.of(TARGET_AUTHORITY, "bar.googleapis.com"), - "cluster.googleapis.com"), - buildVirtualHostV2( - ImmutableList.of("something does not match"), - "some cluster")))) - .build())))); - DiscoveryResponse response = - buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); - responseObserver.onNext(response); - - assertThat(ldsRespTimer.isCancelled()).isTrue(); - - // Client sends an ACK request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); - - ArgumentCaptor configUpdateCaptor = ArgumentCaptor.forClass(null); - verify(configWatcher).onConfigChanged(configUpdateCaptor.capture()); - assertConfigUpdateContainsSingleClusterRoute( - configUpdateCaptor.getValue(), "cluster.googleapis.com"); - - verifyNoMoreInteractions(requestObserver); - } - - /** - * Client receives an RDS response (after a previous LDS request-response) that does not contain a - * RouteConfiguration for the requested resource while each received RouteConfiguration is valid. - * The RDS response is ACKed. - * After the resource fetch timeout expires, watcher waiting for the resource is notified - * with resource unavailable. - */ - @Test - public void rdsResponseWithoutMatchingResource() { - xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - // Client sends an LDS request for the host name (with port) to management server. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - - Rds rdsConfig = - Rds.newBuilder() - // Must set to use ADS. - .setConfigSource( - ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) - .setRouteConfigName("route-foo.googleapis.com") - .build(); - List listeners = ImmutableList.of( - Any.pack(buildListenerV2(TARGET_AUTHORITY, /* matching resource */ - Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) - ); - DiscoveryResponse response = - buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sends an ACK LDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); - - // Client sends an (first) RDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, ""))); - - assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); - - // Management server should only sends RouteConfiguration messages with at least one - // VirtualHost with domains matching requested hostname. Otherwise, it is invalid data. - List routeConfigs = ImmutableList.of( - Any.pack( - buildRouteConfigurationV2( - "some resource name does not match route-foo.googleapis.com", - ImmutableList.of( - buildVirtualHostV2( - ImmutableList.of(TARGET_AUTHORITY), - "whatever cluster")))), - Any.pack( - buildRouteConfigurationV2( - "some other resource name does not match route-foo.googleapis.com", - ImmutableList.of( - buildVirtualHostV2( - ImmutableList.of(TARGET_AUTHORITY), - "some more whatever cluster"))))); - response = buildDiscoveryResponseV2( - "0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sends an ACK RDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); - - verify(configWatcher, never()).onConfigChanged(any(ConfigUpdate.class)); - verify(configWatcher, never()).onResourceDoesNotExist(anyString()); - verify(configWatcher, never()).onError(any(Status.class)); - fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); - verify(configWatcher).onResourceDoesNotExist("route-foo.googleapis.com"); - assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); - } - - /** - * Client resolves the virtual host config from an RDS response for the requested resource. The - * RDS response is ACKed. - * The config watcher is notified with an update. - */ - @Test - public void resolveVirtualHostInRdsResponse() { - xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - Rds rdsConfig = - Rds.newBuilder() - // Must set to use ADS. - .setConfigSource( - ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) - .setRouteConfigName("route-foo.googleapis.com") - .build(); - - List listeners = ImmutableList.of( - Any.pack(buildListenerV2(TARGET_AUTHORITY, /* matching resource */ - Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) - ); - DiscoveryResponse response = - buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sends an ACK LDS request and an RDS request for "route-foo.googleapis.com". (Omitted) - - assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); - - // Management server should only sends RouteConfiguration messages with at least one - // VirtualHost with domains matching requested hostname. Otherwise, it is invalid data. - List routeConfigs = ImmutableList.of( - Any.pack( - buildRouteConfigurationV2( - "route-foo.googleapis.com", // target route configuration - ImmutableList.of( - buildVirtualHostV2(ImmutableList.of("something does not match"), - "some cluster"), - buildVirtualHostV2(ImmutableList.of(TARGET_AUTHORITY, "bar.googleapis.com:443"), - "cluster.googleapis.com")))), // matching virtual host - Any.pack( - buildRouteConfigurationV2( - "some resource name does not match route-foo.googleapis.com", - ImmutableList.of( - buildVirtualHostV2(ImmutableList.of("foo.googleapis.com"), - "some more cluster"))))); - response = buildDiscoveryResponseV2( - "0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); - responseObserver.onNext(response); - - assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); - - // Client sent an ACK RDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); - - ArgumentCaptor configUpdateCaptor = ArgumentCaptor.forClass(null); - verify(configWatcher).onConfigChanged(configUpdateCaptor.capture()); - assertConfigUpdateContainsSingleClusterRoute( - configUpdateCaptor.getValue(), "cluster.googleapis.com"); - } - - /** - * Client resolves the virtual host config with path matching from an RDS response for the - * requested resource. The RDS response is ACKed. - * The config watcher is notified with an update. - */ - @Test - public void resolveVirtualHostWithPathMatchingInRdsResponse() { - xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - Rds rdsConfig = - Rds.newBuilder() - // Must set to use ADS. - .setConfigSource( - ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) - .setRouteConfigName("route-foo.googleapis.com") - .build(); - - List listeners = ImmutableList.of( - Any.pack(buildListenerV2(TARGET_AUTHORITY, /* matching resource */ - Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) - ); - DiscoveryResponse response = - buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sends an ACK LDS request and an RDS request for "route-foo.googleapis.com". (Omitted) - - assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); - - // Management server should only sends RouteConfiguration messages with at least one - // VirtualHost with domains matching requested hostname. Otherwise, it is invalid data. - List routeConfigs = - ImmutableList.of( - Any.pack( - buildRouteConfigurationV2( - "route-foo.googleapis.com", - ImmutableList.of( - io.envoyproxy.envoy.api.v2.route.VirtualHost.newBuilder() - .setName("virtualhost00.googleapis.com") // don't care - // domains wit a match. - .addAllDomains(ImmutableList.of(TARGET_AUTHORITY, "bar.googleapis.com")) - .addRoutes( - io.envoyproxy.envoy.api.v2.route.Route.newBuilder() - // path match with cluster route - .setRoute( - io.envoyproxy.envoy.api.v2.route.RouteAction.newBuilder() - .setCluster("cl1.googleapis.com")) - .setMatch( - io.envoyproxy.envoy.api.v2.route.RouteMatch.newBuilder() - .setPath("/service1/method1"))) - .addRoutes( - io.envoyproxy.envoy.api.v2.route.Route.newBuilder() - // path match with weighted cluster route - .setRoute( - io.envoyproxy.envoy.api.v2.route.RouteAction.newBuilder() - .setWeightedClusters( - WeightedCluster.newBuilder() - .addClusters( - WeightedCluster.ClusterWeight.newBuilder() - .setWeight(UInt32Value.of(30)) - .setName("cl21.googleapis.com")) - .addClusters( - WeightedCluster.ClusterWeight.newBuilder() - .setWeight(UInt32Value.of(70)) - .setName("cl22.googleapis.com")))) - .setMatch( - io.envoyproxy.envoy.api.v2.route.RouteMatch.newBuilder() - .setPath("/service2/method2"))) - .addRoutes( - io.envoyproxy.envoy.api.v2.route.Route.newBuilder() - // prefix match with cluster route - .setRoute( - io.envoyproxy.envoy.api.v2.route.RouteAction.newBuilder() - .setCluster("cl1.googleapis.com")) - .setMatch( - io.envoyproxy.envoy.api.v2.route.RouteMatch.newBuilder() - .setPrefix("/service1/"))) - .addRoutes( - io.envoyproxy.envoy.api.v2.route.Route.newBuilder() - // default match with cluster route - .setRoute( - io.envoyproxy.envoy.api.v2.route.RouteAction.newBuilder() - .setCluster("cluster.googleapis.com")) - .setMatch( - io.envoyproxy.envoy.api.v2.route.RouteMatch.newBuilder() - .setPrefix(""))) - .build())))); - response = buildDiscoveryResponseV2( - "0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); - responseObserver.onNext(response); - - assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); - - // Client sent an ACK RDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); - - ArgumentCaptor configUpdateCaptor = ArgumentCaptor.forClass(null); - verify(configWatcher).onConfigChanged(configUpdateCaptor.capture()); - List routes = configUpdateCaptor.getValue().getRoutes(); - assertThat(routes).hasSize(4); - assertThat(routes.get(0)) - .isEqualTo( - new EnvoyProtoData.Route( - // path match with cluster route - new io.grpc.xds.RouteMatch( - /* pathPrefixMatch= */ null, /* pathExactMatch= */ "/service1/method1"), - new EnvoyProtoData.RouteAction( - TimeUnit.SECONDS.toNanos(15L), "cl1.googleapis.com", null))); - assertThat(routes.get(1)) - .isEqualTo( - new EnvoyProtoData.Route( - // path match with weighted cluster route - new io.grpc.xds.RouteMatch( - /* pathPrefixMatch= */ null, /* pathExactMatch= */ "/service2/method2"), - new EnvoyProtoData.RouteAction( - TimeUnit.SECONDS.toNanos(15L), - null, - ImmutableList.of( - new EnvoyProtoData.ClusterWeight("cl21.googleapis.com", 30), - new EnvoyProtoData.ClusterWeight("cl22.googleapis.com", 70))))); - assertThat(routes.get(2)) - .isEqualTo( - new EnvoyProtoData.Route( - // prefix match with cluster route - new io.grpc.xds.RouteMatch( - /* pathPrefixMatch= */ "/service1/", /* pathExactMatch= */ null), - new EnvoyProtoData.RouteAction( - TimeUnit.SECONDS.toNanos(15L), "cl1.googleapis.com", null))); - assertThat(routes.get(3)) - .isEqualTo( - new EnvoyProtoData.Route( - // default match with cluster route - new io.grpc.xds.RouteMatch( - /* pathPrefixMatch= */ "", /* pathExactMatch= */ null), - new EnvoyProtoData.RouteAction( - TimeUnit.SECONDS.toNanos(15L), "cluster.googleapis.com", null))); - } - - /** - * Client receives an RDS response (after a previous LDS request-response) containing a - * RouteConfiguration message for the requested resource. But the RouteConfiguration message - * is invalid as it does not contain any VirtualHost with domains matching the requested - * hostname. - * The RDS response is NACKed, as if the XdsClient has not received this response. - */ - @Test - public void failToFindVirtualHostInRdsResponse() { - xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - Rds rdsConfig = - Rds.newBuilder() - // Must set to use ADS. - .setConfigSource( - ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) - .setRouteConfigName("route-foo.googleapis.com") - .build(); - - List listeners = ImmutableList.of( - Any.pack(buildListenerV2(TARGET_AUTHORITY, /* matching resource */ - Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) - ); - DiscoveryResponse response = - buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sends an ACK LDS request and an RDS request for "route-foo.googleapis.com". (Omitted) - - assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); - - List routeConfigs = ImmutableList.of( - Any.pack( - buildRouteConfigurationV2( - "route-foo.googleapis.com", - ImmutableList.of( - buildVirtualHostV2(ImmutableList.of("something does not match"), - "some cluster"), - buildVirtualHostV2( - ImmutableList.of("something else does not match", "also does not match"), - "cluster.googleapis.com")))), - Any.pack( - buildRouteConfigurationV2( - "some resource name does not match route-foo.googleapis.com", - ImmutableList.of( - buildVirtualHostV2(ImmutableList.of("one more does not match"), - "some more cluster"))))); - response = buildDiscoveryResponseV2( - "0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sent an NACK RDS request. - verify(requestObserver) - .onNext( - argThat(new DiscoveryRequestMatcher("", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); - - verify(configWatcher, never()).onConfigChanged(any(ConfigUpdate.class)); - verify(configWatcher, never()).onResourceDoesNotExist(anyString()); - verify(configWatcher, never()).onError(any(Status.class)); - fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); - verify(configWatcher).onResourceDoesNotExist("route-foo.googleapis.com"); - assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); - } - - /** - * Client receives an RDS response (after a previous LDS request-response) containing a - * RouteConfiguration message for the requested resource. But the RouteConfiguration message - * is invalid as the VirtualHost with domains matching the requested hostname contains invalid - * data, its RouteAction message is absent. - * The RDS response is NACKed, as if the XdsClient has not received this response. - */ - @Test - public void matchingVirtualHostDoesNotContainRouteAction() { - xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - Rds rdsConfig = - Rds.newBuilder() - // Must set to use ADS. - .setConfigSource( - ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) - .setRouteConfigName("route-foo.googleapis.com") - .build(); - - List listeners = ImmutableList.of( - Any.pack(buildListenerV2(TARGET_AUTHORITY, /* matching resource */ - Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) - ); - DiscoveryResponse response = - buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sends an ACK LDS request and an RDS request for "route-foo.googleapis.com". (Omitted) - - assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); - - // A VirtualHost with a Route that contains only redirect configuration. - io.envoyproxy.envoy.api.v2.route.VirtualHost virtualHost = - io.envoyproxy.envoy.api.v2.route.VirtualHost.newBuilder() - .setName("virtualhost00.googleapis.com") // don't care - .addDomains(TARGET_AUTHORITY) - .addRoutes( - io.envoyproxy.envoy.api.v2.route.Route.newBuilder() - .setRedirect( - RedirectAction.newBuilder() - .setHostRedirect("bar.googleapis.com") - .setPortRedirect(443))) - .build(); - - List routeConfigs = ImmutableList.of( - Any.pack( - buildRouteConfigurationV2("route-foo.googleapis.com", - ImmutableList.of(virtualHost)))); - response = buildDiscoveryResponseV2( - "0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sent an NACK RDS request. - verify(requestObserver) - .onNext( - argThat(new DiscoveryRequestMatcher("", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); - - verify(configWatcher, never()).onConfigChanged(any(ConfigUpdate.class)); - verify(configWatcher, never()).onResourceDoesNotExist(anyString()); - verify(configWatcher, never()).onError(any(Status.class)); - fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); - verify(configWatcher).onResourceDoesNotExist("route-foo.googleapis.com"); - assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); - } - - /** - * Client receives LDS/RDS responses for updating resources previously received. - * - *

Tests for streaming behavior. - */ - @Test - public void notifyUpdatedResources() { - xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - // Client sends an LDS request for the host name (with port) to management server. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - - // Management server sends back an LDS response containing a RouteConfiguration for the - // requested Listener directly in-line. - io.envoyproxy.envoy.api.v2.RouteConfiguration routeConfig = - buildRouteConfigurationV2( - "route-foo.googleapis.com", // target route configuration - ImmutableList.of( - buildVirtualHostV2( // matching virtual host - ImmutableList.of(TARGET_AUTHORITY, "bar.googleapis.com:443"), - "cluster.googleapis.com"), - buildVirtualHostV2(ImmutableList.of("something does not match"), - "some cluster"))); - - List listeners = ImmutableList.of( - Any.pack(buildListenerV2(TARGET_AUTHORITY, /* matching resource */ - Any.pack(HttpConnectionManager.newBuilder().setRouteConfig(routeConfig).build()))) - ); - DiscoveryResponse response = - buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sends an ACK LDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); - - // Cluster name is resolved and notified to config watcher. - ArgumentCaptor configUpdateCaptor = ArgumentCaptor.forClass(null); - verify(configWatcher).onConfigChanged(configUpdateCaptor.capture()); - assertConfigUpdateContainsSingleClusterRoute( - configUpdateCaptor.getValue(), "cluster.googleapis.com"); - - // Management sends back another LDS response containing updates for the requested Listener. - routeConfig = - buildRouteConfigurationV2( - "another-route-foo.googleapis.com", - ImmutableList.of( - buildVirtualHostV2(ImmutableList.of(TARGET_AUTHORITY, "bar.googleapis.com:443"), - "another-cluster.googleapis.com"), - buildVirtualHostV2(ImmutableList.of("something does not match"), - "some cluster"))); - - listeners = ImmutableList.of( - Any.pack(buildListenerV2(TARGET_AUTHORITY, /* matching resource */ - Any.pack(HttpConnectionManager.newBuilder().setRouteConfig(routeConfig).build()))) - ); - response = - buildDiscoveryResponseV2("1", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0001"); - responseObserver.onNext(response); - - // Client sends an ACK LDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "1", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0001"))); - - // Updated cluster name is notified to config watcher. - configUpdateCaptor = ArgumentCaptor.forClass(null); - verify(configWatcher, times(2)).onConfigChanged(configUpdateCaptor.capture()); - assertConfigUpdateContainsSingleClusterRoute( - configUpdateCaptor.getValue(), "another-cluster.googleapis.com"); - - // Management server sends back another LDS response containing updates for the requested - // Listener and telling client to do RDS. - Rds rdsConfig = - Rds.newBuilder() - // Must set to use ADS. - .setConfigSource( - ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) - .setRouteConfigName("some-route-to-foo.googleapis.com") - .build(); - - listeners = ImmutableList.of( - Any.pack(buildListenerV2(TARGET_AUTHORITY, /* matching resource */ - Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) - ); - response = - buildDiscoveryResponseV2("2", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0002"); - responseObserver.onNext(response); - - // Client sends an ACK LDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "2", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0002"))); - - // Client sends an (first) RDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "some-route-to-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, ""))); - - // Management server sends back an RDS response containing the RouteConfiguration - // for the requested resource. - List routeConfigs = ImmutableList.of( - Any.pack( - buildRouteConfigurationV2( - "some-route-to-foo.googleapis.com", - ImmutableList.of( - buildVirtualHostV2(ImmutableList.of("something does not match"), - "some cluster"), - buildVirtualHostV2(ImmutableList.of(TARGET_AUTHORITY, "bar.googleapis.com:443"), - "some-other-cluster.googleapis.com"))))); - response = buildDiscoveryResponseV2( - "0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sent an ACK RDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "some-route-to-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); - - // Updated cluster name is notified to config watcher again. - configUpdateCaptor = ArgumentCaptor.forClass(null); - verify(configWatcher, times(3)).onConfigChanged(configUpdateCaptor.capture()); - assertConfigUpdateContainsSingleClusterRoute( - configUpdateCaptor.getValue(), "some-other-cluster.googleapis.com"); - - // Management server sends back another RDS response containing updated information for the - // RouteConfiguration currently in-use by client. - routeConfigs = ImmutableList.of( - Any.pack( - buildRouteConfigurationV2( - "some-route-to-foo.googleapis.com", - ImmutableList.of( - buildVirtualHostV2(ImmutableList.of(TARGET_AUTHORITY, "bar.googleapis.com:443"), - "an-updated-cluster.googleapis.com"))))); - response = buildDiscoveryResponseV2( - "1", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0001"); - responseObserver.onNext(response); - - // Client sent an ACK RDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "1", "some-route-to-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0001"))); - - // Updated cluster name is notified to config watcher again. - configUpdateCaptor = ArgumentCaptor.forClass(null); - verify(configWatcher, times(4)).onConfigChanged(configUpdateCaptor.capture()); - assertConfigUpdateContainsSingleClusterRoute( - configUpdateCaptor.getValue(), "an-updated-cluster.googleapis.com"); - - // Management server sends back an LDS response indicating all Listener resources are removed. - response = - buildDiscoveryResponseV2("3", ImmutableList.of(), - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0003"); - responseObserver.onNext(response); - - verify(configWatcher).onResourceDoesNotExist(TARGET_AUTHORITY); - } - - // TODO(chengyuanzhang): tests for timeout waiting for responses for incremental - // protocols (RDS/EDS). - - /** - * Client receives multiple RDS responses without RouteConfiguration for the requested - * resource. It should continue waiting until such an RDS response arrives, as RDS - * protocol is incremental. - * - *

Tests for RDS incremental protocol behavior. - */ - @Test - public void waitRdsResponsesForRequestedResource() { - xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - // Client sends an LDS request for the host name (with port) to management server. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - - // Management sends back an LDS response telling client to do RDS. - Rds rdsConfig = - Rds.newBuilder() - // Must set to use ADS. - .setConfigSource( - ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) - .setRouteConfigName("route-foo.googleapis.com") - .build(); - - List listeners = ImmutableList.of( - Any.pack(buildListenerV2(TARGET_AUTHORITY, /* matching resource */ - Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) - ); - DiscoveryResponse response = - buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sends an ACK LDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); - - // Client sends an (first) RDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, ""))); - - ScheduledTask rdsRespTimer = - Iterables.getOnlyElement( - fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); - assertThat(rdsRespTimer.isCancelled()).isFalse(); - - fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC - 2, TimeUnit.SECONDS); - - // Management server sends back an RDS response that does not contain RouteConfiguration - // for the requested resource. - List routeConfigs = ImmutableList.of( - Any.pack( - buildRouteConfigurationV2( - "some resource name does not match route-foo.googleapis.com", - ImmutableList.of( - buildVirtualHostV2( - ImmutableList.of(TARGET_AUTHORITY), - "some more cluster"))))); - response = buildDiscoveryResponseV2( - "0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sent an ACK RDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); - - // Client waits for future RDS responses silently. - verifyNoMoreInteractions(configWatcher); - assertThat(rdsRespTimer.isCancelled()).isFalse(); - - fakeClock.forwardTime(1, TimeUnit.SECONDS); - - // Management server sends back another RDS response containing the RouteConfiguration - // for the requested resource. - routeConfigs = ImmutableList.of( - Any.pack( - buildRouteConfigurationV2( - "route-foo.googleapis.com", // target route configuration - ImmutableList.of( - buildVirtualHostV2( - ImmutableList.of("something does not match"), - "some cluster"), - buildVirtualHostV2( // matching virtual host - ImmutableList.of(TARGET_AUTHORITY, "bar.googleapis.com:443"), - "another-cluster.googleapis.com"))))); - response = buildDiscoveryResponseV2( - "1", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0001"); - responseObserver.onNext(response); - - // Client sent an ACK RDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "1", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0001"))); - - // Updated cluster name is notified to config watcher. - ArgumentCaptor configUpdateCaptor = ArgumentCaptor.forClass(null); - verify(configWatcher).onConfigChanged(configUpdateCaptor.capture()); - assertConfigUpdateContainsSingleClusterRoute( - configUpdateCaptor.getValue(), "another-cluster.googleapis.com"); - assertThat(rdsRespTimer.isCancelled()).isTrue(); - } - - /** - * An RouteConfiguration is removed by server by sending client an LDS response removing the - * corresponding Listener. - */ - @Test - public void routeConfigurationRemovedNotifiedToWatcher() { - xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - // Client sends an LDS request for the host name (with port) to management server. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - - // Management sends back an LDS response telling client to do RDS. - Rds rdsConfig = - Rds.newBuilder() - // Must set to use ADS. - .setConfigSource( - ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) - .setRouteConfigName("route-foo.googleapis.com") - .build(); - - List listeners = ImmutableList.of( - Any.pack(buildListenerV2(TARGET_AUTHORITY, /* matching resource */ - Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) - ); - DiscoveryResponse response = - buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sends an ACK LDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); - - // Client sends an (first) RDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, ""))); - - // Management server sends back an RDS response containing RouteConfiguration requested. - List routeConfigs = ImmutableList.of( - Any.pack( - buildRouteConfigurationV2( - "route-foo.googleapis.com", // target route configuration - ImmutableList.of( - buildVirtualHostV2( - ImmutableList.of(TARGET_AUTHORITY), // matching virtual host - "cluster.googleapis.com"))))); - response = buildDiscoveryResponseV2( - "0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sent an ACK RDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"))); - - // Resolved cluster name is notified to config watcher. - ArgumentCaptor configUpdateCaptor = ArgumentCaptor.forClass(null); - verify(configWatcher).onConfigChanged(configUpdateCaptor.capture()); - assertConfigUpdateContainsSingleClusterRoute( - configUpdateCaptor.getValue(), "cluster.googleapis.com"); - - // Management server sends back another LDS response with the previous Listener (currently - // in-use by client) removed as the RouteConfiguration it references to is absent. - response = - buildDiscoveryResponseV2("1", ImmutableList.of(), // empty - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0001"); - responseObserver.onNext(response); - - // Client sent an ACK LDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "1", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0001"))); - - verify(configWatcher).onResourceDoesNotExist(TARGET_AUTHORITY); - } - - /** - * Management server sends another LDS response for updating the RDS resource to be requested - * while client is currently requesting for a previously given RDS resource name. - */ - @Test - public void updateRdsRequestResourceWhileInitialResourceFetchInProgress() { - xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - // Management sends back an LDS response telling client to do RDS. - Rds rdsConfig = - Rds.newBuilder() - // Must set to use ADS. - .setConfigSource( - ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) - .setRouteConfigName("route-foo.googleapis.com") - .build(); - - List listeners = ImmutableList.of( - Any.pack(buildListenerV2(TARGET_AUTHORITY, /* matching resource */ - Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) - ); - DiscoveryResponse response = - buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sends an (first) RDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, ""))); - - ScheduledTask rdsRespTimer = - Iterables.getOnlyElement( - fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); - assertThat(rdsRespTimer.isCancelled()).isFalse(); - - // Management sends back another LDS response updating the Listener information to use - // another resource name for doing RDS. - rdsConfig = - Rds.newBuilder() - // Must set to use ADS. - .setConfigSource( - ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) - .setRouteConfigName("route-bar.googleapis.com") - .build(); - - listeners = ImmutableList.of( - Any.pack( - buildListenerV2( - TARGET_AUTHORITY, /* matching resource */ - Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) - ); - response = buildDiscoveryResponseV2("1", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0001"); - responseObserver.onNext(response); - - // Client sent a new RDS request with updated resource name. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "route-bar.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, ""))); - - assertThat(rdsRespTimer.isCancelled()).isTrue(); - rdsRespTimer = - Iterables.getOnlyElement( - fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); - assertThat(rdsRespTimer.isCancelled()).isFalse(); - - // Management server sends back an RDS response containing RouteConfiguration requested. - List routeConfigs = ImmutableList.of( - Any.pack( - buildRouteConfigurationV2( - "route-bar.googleapis.com", // target route configuration - ImmutableList.of( - buildVirtualHostV2( - ImmutableList.of(TARGET_AUTHORITY), // matching virtual host - "cluster.googleapis.com"))))); - response = buildDiscoveryResponseV2( - "0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); - responseObserver.onNext(response); - - assertThat(rdsRespTimer.isCancelled()).isTrue(); - } - - /** - * Client receives an CDS response that does not contain a Cluster for the requested resource - * while each received Cluster is valid. The CDS response is ACKed. Cluster watchers are notified - * with resource unavailable after initial resource fetch timeout has expired. - */ - @Test - public void cdsResponseWithoutMatchingResource() { - xdsClient.watchCdsResource("cluster-foo.googleapis.com", cdsResourceWatcher); - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - // Client sends a CDS request for the only cluster being watched to management server. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); - assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); - - // Management server sends back a CDS response without Cluster for the requested resource. - List clusters = ImmutableList.of( - Any.pack(buildClusterV2("cluster-bar.googleapis.com", null, false)), - Any.pack(buildClusterV2("cluster-baz.googleapis.com", null, false))); - DiscoveryResponse response = - buildDiscoveryResponseV2("0", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sent an ACK CDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); - verify(cdsResourceWatcher, never()).onChanged(any(CdsUpdate.class)); - verify(cdsResourceWatcher, never()).onResourceDoesNotExist("cluster-foo.googleapis.com"); - verify(cdsResourceWatcher, never()).onError(any(Status.class)); - - fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); - verify(cdsResourceWatcher).onResourceDoesNotExist("cluster-foo.googleapis.com"); - assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); - } - - /** - * Normal workflow of receiving a CDS response containing Cluster message for a requested - * cluster. - */ - @Test - public void cdsResponseWithMatchingResource() { - xdsClient.watchCdsResource("cluster-foo.googleapis.com", cdsResourceWatcher); - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - // Client sends a CDS request for the only cluster being watched to management server. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); - ScheduledTask cdsRespTimer = - Iterables.getOnlyElement( - fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); - - // Management server sends back a CDS response without Cluster for the requested resource. - List clusters = ImmutableList.of( - Any.pack(buildClusterV2("cluster-bar.googleapis.com", null, false)), - Any.pack(buildClusterV2("cluster-foo.googleapis.com", null, false)), - Any.pack(buildClusterV2("cluster-baz.googleapis.com", null, false))); - DiscoveryResponse response = - buildDiscoveryResponseV2("0", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sent an ACK CDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); - assertThat(cdsRespTimer.isCancelled()).isTrue(); - - ArgumentCaptor cdsUpdateCaptor = ArgumentCaptor.forClass(null); - verify(cdsResourceWatcher).onChanged(cdsUpdateCaptor.capture()); - CdsUpdate cdsUpdate = cdsUpdateCaptor.getValue(); - assertThat(cdsUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(cdsUpdate.getEdsServiceName()).isNull(); - assertThat(cdsUpdate.getLbPolicy()).isEqualTo("round_robin"); - assertThat(cdsUpdate.getLrsServerName()).isNull(); - - // Management server sends back another CDS response updating the requested Cluster. - clusters = ImmutableList.of( - Any.pack(buildClusterV2("cluster-bar.googleapis.com", null, false)), - Any.pack( - buildClusterV2("cluster-foo.googleapis.com", "eds-cluster-foo.googleapis.com", true)), - Any.pack(buildClusterV2("cluster-baz.googleapis.com", null, false))); - response = - buildDiscoveryResponseV2("1", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"); - responseObserver.onNext(response); - - // Client sent an ACK CDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "1", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"))); - - verify(cdsResourceWatcher, times(2)).onChanged(cdsUpdateCaptor.capture()); - cdsUpdate = cdsUpdateCaptor.getValue(); - assertThat(cdsUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(cdsUpdate.getEdsServiceName()) - .isEqualTo("eds-cluster-foo.googleapis.com"); - assertThat(cdsUpdate.getLbPolicy()).isEqualTo("round_robin"); - assertThat(cdsUpdate.getLrsServerName()).isEqualTo(""); - } - - /** - * CDS response containing UpstreamTlsContext for a cluster. - */ - @Test - public void cdsResponseWithUpstreamTlsContext() { - xdsClient.watchCdsResource("cluster-foo.googleapis.com", cdsResourceWatcher); - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - // Management server sends back CDS response with UpstreamTlsContext. - UpstreamTlsContext testUpstreamTlsContext = - buildUpstreamTlsContextV2("secret1", "unix:/var/uds2"); - List clusters = ImmutableList.of( - Any.pack(buildClusterV2("cluster-bar.googleapis.com", null, false)), - Any.pack(buildSecureClusterV2("cluster-foo.googleapis.com", - "eds-cluster-foo.googleapis.com", true, testUpstreamTlsContext)), - Any.pack(buildClusterV2("cluster-baz.googleapis.com", null, false))); - DiscoveryResponse response = - buildDiscoveryResponseV2("0", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sent an ACK CDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); - ArgumentCaptor cdsUpdateCaptor = ArgumentCaptor.forClass(null); - verify(cdsResourceWatcher, times(1)).onChanged(cdsUpdateCaptor.capture()); - CdsUpdate cdsUpdate = cdsUpdateCaptor.getValue(); - EnvoyServerProtoData.UpstreamTlsContext upstreamTlsContext = cdsUpdate - .getUpstreamTlsContext(); - SdsSecretConfig validationContextSdsSecretConfig = upstreamTlsContext.getCommonTlsContext() - .getValidationContextSdsSecretConfig(); - assertThat(validationContextSdsSecretConfig.getName()).isEqualTo("secret1"); - assertThat( - Iterables.getOnlyElement( - validationContextSdsSecretConfig - .getSdsConfig() - .getApiConfigSource() - .getGrpcServicesList()) - .getGoogleGrpc() - .getTargetUri()) - .isEqualTo("unix:/var/uds2"); - } - - @Test - public void multipleCdsWatchers() { - CdsResourceWatcher watcher1 = mock(CdsResourceWatcher.class); - CdsResourceWatcher watcher2 = mock(CdsResourceWatcher.class); - CdsResourceWatcher watcher3 = mock(CdsResourceWatcher.class); - xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher1); - xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher2); - xdsClient.watchCdsResource("cluster-bar.googleapis.com", watcher3); - - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - // Client sends a CDS request containing all clusters being watched to management server. - verify(requestObserver) - .onNext( - argThat( - new DiscoveryRequestMatcher("", - ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); - assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(2); - - // Management server sends back a CDS response contains Cluster for only one of - // requested cluster. - List clusters = ImmutableList.of( - Any.pack(buildClusterV2("cluster-foo.googleapis.com", null, false))); - DiscoveryResponse response = - buildDiscoveryResponseV2("0", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"); - responseObserver.onNext(response); - - assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); - // Client sent an ACK CDS request. - verify(requestObserver) - .onNext( - argThat( - new DiscoveryRequestMatcher("0", - ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); - - // Two watchers get notification of cluster update for the cluster they are interested in. - ArgumentCaptor cdsUpdateCaptor1 = ArgumentCaptor.forClass(null); - verify(watcher1).onChanged(cdsUpdateCaptor1.capture()); - CdsUpdate cdsUpdate1 = cdsUpdateCaptor1.getValue(); - assertThat(cdsUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(cdsUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(cdsUpdate1.getEdsServiceName()).isNull(); - assertThat(cdsUpdate1.getLbPolicy()).isEqualTo("round_robin"); - assertThat(cdsUpdate1.getLrsServerName()).isNull(); - - ArgumentCaptor cdsUpdateCaptor2 = ArgumentCaptor.forClass(null); - verify(watcher2).onChanged(cdsUpdateCaptor2.capture()); - CdsUpdate cdsUpdate2 = cdsUpdateCaptor2.getValue(); - assertThat(cdsUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(cdsUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(cdsUpdate2.getEdsServiceName()).isNull(); - assertThat(cdsUpdate2.getLbPolicy()).isEqualTo("round_robin"); - assertThat(cdsUpdate2.getLrsServerName()).isNull(); - - verify(watcher3, never()).onChanged(any(CdsUpdate.class)); - verify(watcher3, never()).onResourceDoesNotExist("cluster-bar.googleapis.com"); - verify(watcher3, never()).onError(any(Status.class)); - - // The other watcher gets an error notification for cluster not found after its timer expired. - fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); - verify(watcher3).onResourceDoesNotExist("cluster-bar.googleapis.com"); - assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); - - // Management server sends back another CDS response contains Clusters for all - // requested clusters. - clusters = ImmutableList.of( - Any.pack(buildClusterV2("cluster-foo.googleapis.com", null, false)), - Any.pack( - buildClusterV2("cluster-bar.googleapis.com", - "eds-cluster-bar.googleapis.com", true))); - response = buildDiscoveryResponseV2("1", clusters, - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"); - responseObserver.onNext(response); - - // Client sent an ACK CDS request. - verify(requestObserver) - .onNext( - argThat( - new DiscoveryRequestMatcher("1", - ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"))); - - verifyNoMoreInteractions(watcher1, watcher2); // resource has no change - ArgumentCaptor cdsUpdateCaptor3 = ArgumentCaptor.forClass(null); - verify(watcher3).onChanged(cdsUpdateCaptor3.capture()); - CdsUpdate cdsUpdate3 = cdsUpdateCaptor3.getValue(); - assertThat(cdsUpdate3.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); - assertThat(cdsUpdate3.getEdsServiceName()) - .isEqualTo("eds-cluster-bar.googleapis.com"); - assertThat(cdsUpdate3.getLbPolicy()).isEqualTo("round_robin"); - assertThat(cdsUpdate3.getLrsServerName()).isEqualTo(""); - } - - /** - * (CDS response caching behavior) Adding cluster watchers interested in some cluster that - * some other endpoint watcher had already been watching on will result in cluster update - * notified to the newly added watcher immediately, without sending new CDS requests. - */ - @Test - public void watchClusterAlreadyBeingWatched() { - CdsResourceWatcher watcher1 = mock(CdsResourceWatcher.class); - xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher1); - - // Streaming RPC starts after a first watcher is added. - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - // Client sends an CDS request to management server. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); - assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); - - // Management server sends back an CDS response with Cluster for the requested - // cluster. - List clusters = ImmutableList.of( - Any.pack(buildClusterV2("cluster-foo.googleapis.com", null, false))); - DiscoveryResponse response = - buildDiscoveryResponseV2("0", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sent an ACK CDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); - - ArgumentCaptor cdsUpdateCaptor1 = ArgumentCaptor.forClass(null); - verify(watcher1).onChanged(cdsUpdateCaptor1.capture()); - CdsUpdate cdsUpdate1 = cdsUpdateCaptor1.getValue(); - assertThat(cdsUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(cdsUpdate1.getEdsServiceName()).isNull(); - assertThat(cdsUpdate1.getLbPolicy()).isEqualTo("round_robin"); - assertThat(cdsUpdate1.getLrsServerName()).isNull(); - assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); - - // Another cluster watcher interested in the same cluster is added. - CdsResourceWatcher watcher2 = mock(CdsResourceWatcher.class); - xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher2); - - // Since the client has received cluster update for this cluster before, cached result is - // notified to the newly added watcher immediately. - ArgumentCaptor cdsUpdateCaptor2 = ArgumentCaptor.forClass(null); - verify(watcher2).onChanged(cdsUpdateCaptor2.capture()); - CdsUpdate cdsUpdate2 = cdsUpdateCaptor2.getValue(); - assertThat(cdsUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(cdsUpdate2.getEdsServiceName()).isNull(); - assertThat(cdsUpdate2.getLbPolicy()).isEqualTo("round_robin"); - assertThat(cdsUpdate2.getLrsServerName()).isNull(); - - verifyNoMoreInteractions(requestObserver); - assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); - } - - /** - * Basic operations of adding/canceling cluster data watchers. - */ - @Test - public void addRemoveCdsWatchers() { - CdsResourceWatcher watcher1 = mock(CdsResourceWatcher.class); - xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher1); - - // Streaming RPC starts after a first watcher is added. - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - // Client sends an CDS request to management server. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); - - // Management server sends back a CDS response with Cluster for the requested - // cluster. - List clusters = ImmutableList.of( - Any.pack(buildClusterV2("cluster-foo.googleapis.com", null, false))); - DiscoveryResponse response = - buildDiscoveryResponseV2("0", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sent an ACK CDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); - - ArgumentCaptor cdsUpdateCaptor1 = ArgumentCaptor.forClass(null); - verify(watcher1).onChanged(cdsUpdateCaptor1.capture()); - CdsUpdate cdsUpdate1 = cdsUpdateCaptor1.getValue(); - assertThat(cdsUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(cdsUpdate1.getEdsServiceName()).isNull(); - assertThat(cdsUpdate1.getLbPolicy()).isEqualTo("round_robin"); - assertThat(cdsUpdate1.getLrsServerName()).isNull(); - - // Add another cluster watcher for a different cluster. - CdsResourceWatcher watcher2 = mock(CdsResourceWatcher.class); - xdsClient.watchCdsResource("cluster-bar.googleapis.com", watcher2); - - // Client sent a new CDS request for all interested resources. - verify(requestObserver) - .onNext( - argThat( - new DiscoveryRequestMatcher("0", - ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"))); - - // Management server sends back a CDS response with Cluster for all requested cluster. - clusters = ImmutableList.of( - Any.pack(buildClusterV2("cluster-foo.googleapis.com", null, false)), - Any.pack( - buildClusterV2("cluster-bar.googleapis.com", - "eds-cluster-bar.googleapis.com", true))); - response = buildDiscoveryResponseV2("1", clusters, - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"); - responseObserver.onNext(response); - - // Client sent an ACK CDS request for all interested resources. - verify(requestObserver) - .onNext( - argThat( - new DiscoveryRequestMatcher("1", - ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"))); - verifyNoMoreInteractions(watcher1); // resource has no change - ArgumentCaptor cdsUpdateCaptor2 = ArgumentCaptor.forClass(null); - verify(watcher2).onChanged(cdsUpdateCaptor2.capture()); - CdsUpdate cdsUpdate2 = cdsUpdateCaptor2.getValue(); - assertThat(cdsUpdate2.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); - assertThat(cdsUpdate2.getEdsServiceName()) - .isEqualTo("eds-cluster-bar.googleapis.com"); - assertThat(cdsUpdate2.getLbPolicy()).isEqualTo("round_robin"); - assertThat(cdsUpdate2.getLrsServerName()).isEqualTo(""); - - // Cancel one of the watcher. - xdsClient.cancelCdsResourceWatch("cluster-foo.googleapis.com", watcher1); - - // Since the cancelled watcher was the last watcher interested in that cluster (but there - // is still interested resource), client sent an new CDS request to unsubscribe from - // that cluster. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "1", "cluster-bar.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"))); - - // Management server has nothing to respond. - - // Cancel the other watcher. All resources have been unsubscribed. - xdsClient.cancelCdsResourceWatch("cluster-bar.googleapis.com", watcher2); - - verify(requestObserver) - .onNext( - argThat( - new DiscoveryRequestMatcher("1", ImmutableList.of(), - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"))); - - // Management server sends back a new CDS response. - clusters = ImmutableList.of( - Any.pack(buildClusterV2("cluster-foo.googleapis.com", null, true)), - Any.pack( - buildClusterV2("cluster-bar.googleapis.com", null, false))); - response = - buildDiscoveryResponseV2("2", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0002"); - responseObserver.onNext(response); - - verify(requestObserver) - .onNext( - argThat( - new DiscoveryRequestMatcher("2", ImmutableList.of(), - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0002"))); - - // Cancelled watchers do not receive notification. - verifyNoMoreInteractions(watcher1, watcher2); - - // A new cluster watcher is added to watch cluster foo again. - CdsResourceWatcher watcher3 = mock(CdsResourceWatcher.class); - xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher3); - verify(watcher3, never()).onChanged(any(CdsUpdate.class)); - - // A CDS request is sent to indicate subscription of "cluster-foo.googleapis.com" only. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "2", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0002"))); - - // Management server sends back a new CDS response for at least newly requested resources - // (it is required to do so). - clusters = ImmutableList.of( - Any.pack(buildClusterV2("cluster-foo.googleapis.com", null, true)), - Any.pack( - buildClusterV2("cluster-bar.googleapis.com", null, false))); - response = - buildDiscoveryResponseV2("3", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0003"); - responseObserver.onNext(response); - - // Notified with cached data immediately. - ArgumentCaptor cdsUpdateCaptor3 = ArgumentCaptor.forClass(null); - verify(watcher3).onChanged(cdsUpdateCaptor3.capture()); - CdsUpdate cdsUpdate3 = cdsUpdateCaptor3.getValue(); - assertThat(cdsUpdate3.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(cdsUpdate3.getEdsServiceName()).isNull(); - assertThat(cdsUpdate3.getLbPolicy()).isEqualTo("round_robin"); - assertThat(cdsUpdate2.getLrsServerName()).isEqualTo(""); - - verifyNoMoreInteractions(watcher1, watcher2); - - // A CDS request is sent to re-subscribe the cluster again. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "3", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0003"))); - } - - @Test - public void addRemoveCdsWatcherWhileInitialResourceFetchInProgress() { - CdsResourceWatcher watcher1 = mock(CdsResourceWatcher.class); - xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher1); - - // Streaming RPC starts after a first watcher is added. - StreamObserver requestObserver = requestObservers.poll(); - - // Client sends an EDS request to management server. - verify(requestObserver) - .onNext( - argThat( - new DiscoveryRequestMatcher("", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); - assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); - - fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC - 1, TimeUnit.SECONDS); - - CdsResourceWatcher watcher2 = mock(CdsResourceWatcher.class); - CdsResourceWatcher watcher3 = mock(CdsResourceWatcher.class); - CdsResourceWatcher watcher4 = mock(CdsResourceWatcher.class); - xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher2); - xdsClient.watchCdsResource("cluster-bar.googleapis.com", watcher3); - xdsClient.watchCdsResource("cluster-bar.googleapis.com", watcher4); - - // Client sends a new CDS request for updating the latest resource subscription. - verify(requestObserver) - .onNext( - argThat( - new DiscoveryRequestMatcher("", - ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); - assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(2); - - fakeClock.forwardTime(1, TimeUnit.SECONDS); - assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); - - // CDS resource "cluster-foo.googleapis.com" is known to be absent. - verify(watcher1).onResourceDoesNotExist("cluster-foo.googleapis.com"); - verify(watcher2).onResourceDoesNotExist("cluster-foo.googleapis.com"); - - // The absence result is known immediately. - CdsResourceWatcher watcher5 = mock(CdsResourceWatcher.class); - xdsClient.watchCdsResource("cluster-foo.googleapis.com", watcher5); - verify(watcher5).onResourceDoesNotExist("cluster-foo.googleapis.com"); - - assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); - ScheduledTask timeoutTask = Iterables.getOnlyElement(fakeClock.getPendingTasks()); - - // Cancel watchers while discovery for resource "cluster-bar.googleapis.com" is still - // in progress. - xdsClient.cancelCdsResourceWatch("cluster-bar.googleapis.com", watcher3); - assertThat(timeoutTask.isCancelled()).isFalse(); - xdsClient.cancelCdsResourceWatch("cluster-bar.googleapis.com", watcher4); - - // Client sends a CDS request for resource subscription update (Omitted). - - fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); - - assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); - assertThat(timeoutTask.isCancelled()).isTrue(); - - verifyNoInteractions(watcher3, watcher4); - } - - @Test - public void cdsUpdateForClusterBeingRemoved() { - xdsClient.watchCdsResource("cluster-foo.googleapis.com", cdsResourceWatcher); - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); - assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); - - // Management server sends back a CDS response containing requested resource. - List clusters = ImmutableList.of( - Any.pack(buildClusterV2("cluster-foo.googleapis.com", null, true))); - DiscoveryResponse response = - buildDiscoveryResponseV2("0", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sent an ACK CDS request (Omitted). - - ArgumentCaptor cdsUpdateCaptor = ArgumentCaptor.forClass(null); - verify(cdsResourceWatcher).onChanged(cdsUpdateCaptor.capture()); - CdsUpdate cdsUpdate = cdsUpdateCaptor.getValue(); - assertThat(cdsUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(cdsUpdate.getEdsServiceName()).isNull(); - assertThat(cdsUpdate.getLbPolicy()).isEqualTo("round_robin"); - assertThat(cdsUpdate.getLrsServerName()).isEqualTo(""); - assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); - - // No cluster is available. - response = - buildDiscoveryResponseV2("1", ImmutableList.of(), - XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"); - responseObserver.onNext(response); - - verify(cdsResourceWatcher).onResourceDoesNotExist("cluster-foo.googleapis.com"); - } - - /** - * Client receives an EDS response that does not contain a ClusterLoadAssignment for the - * requested resource while each received ClusterLoadAssignment is valid. - * The EDS response is ACKed. - * After the resource fetch timeout expires, watchers waiting for the resource is notified - * with resource unavailable. - */ - @Test - public void edsResponseWithoutMatchingResource() { - xdsClient.watchEdsResource("cluster-foo.googleapis.com", edsResourceWatcher); - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - // Client sends an EDS request for the only cluster being watched to management server. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); - assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); - - // Management server sends back an EDS response without ClusterLoadAssignment for the requested - // cluster. - List clusterLoadAssignments = ImmutableList.of( - Any.pack(buildClusterLoadAssignmentV2("cluster-bar.googleapis.com", - ImmutableList.of( - buildLocalityLbEndpointsV2("region1", "zone1", "subzone1", - ImmutableList.of( - buildLbEndpointV2("192.168.0.1", 8080, HealthStatus.HEALTHY, 2)), - 1, 0)), - ImmutableList.of())), - Any.pack(buildClusterLoadAssignmentV2("cluster-baz.googleapis.com", - ImmutableList.of( - buildLocalityLbEndpointsV2("region2", "zone2", "subzone2", - ImmutableList.of( - buildLbEndpointV2("192.168.234.52", 8888, HealthStatus.UNKNOWN, 5)), - 6, 1)), - ImmutableList.of()))); - - DiscoveryResponse response = - buildDiscoveryResponseV2("0", clusterLoadAssignments, - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sent an ACK EDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"))); - - verify(edsResourceWatcher, never()).onChanged(any(EdsUpdate.class)); - verify(edsResourceWatcher, never()).onResourceDoesNotExist("cluster-foo.googleapis.com"); - verify(edsResourceWatcher, never()).onError(any(Status.class)); - fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); - verify(edsResourceWatcher).onResourceDoesNotExist("cluster-foo.googleapis.com"); - assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); - } - - /** - * Normal workflow of receiving an EDS response containing ClusterLoadAssignment message for - * a requested cluster. - */ - @Test - public void edsResponseWithMatchingResource() { - xdsClient.watchEdsResource("cluster-foo.googleapis.com", edsResourceWatcher); - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - // Client sends an EDS request for the only cluster being watched to management server. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); - ScheduledTask edsRespTimeoutTask = - Iterables.getOnlyElement( - fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); - assertThat(edsRespTimeoutTask.isCancelled()).isFalse(); - - // Management server sends back an EDS response with ClusterLoadAssignment for the requested - // cluster. - List clusterLoadAssignments = ImmutableList.of( - Any.pack(buildClusterLoadAssignmentV2("cluster-foo.googleapis.com", - ImmutableList.of( - buildLocalityLbEndpointsV2("region1", "zone1", "subzone1", - ImmutableList.of( - buildLbEndpointV2("192.168.0.1", 8080, HealthStatus.HEALTHY, 2)), - 1, 0), - buildLocalityLbEndpointsV2("region3", "zone3", "subzone3", - ImmutableList.of(), - 2, 1), /* locality with 0 endpoint */ - buildLocalityLbEndpointsV2("region4", "zone4", "subzone4", - ImmutableList.of( - buildLbEndpointV2("192.168.142.5", 80, HealthStatus.UNKNOWN, 5)), - 0, 2) /* locality with 0 weight */), - ImmutableList.of( - buildDropOverloadV2("lb", 200), - buildDropOverloadV2("throttle", 1000)))), - Any.pack(buildClusterLoadAssignmentV2("cluster-baz.googleapis.com", - ImmutableList.of( - buildLocalityLbEndpointsV2("region2", "zone2", "subzone2", - ImmutableList.of( - buildLbEndpointV2("192.168.234.52", 8888, HealthStatus.UNKNOWN, 5)), - 6, 1)), - ImmutableList.of()))); - - DiscoveryResponse response = - buildDiscoveryResponseV2("0", clusterLoadAssignments, - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"); - responseObserver.onNext(response); - - assertThat(edsRespTimeoutTask.isCancelled()).isTrue(); - - // Client sent an ACK EDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"))); - - ArgumentCaptor edsUpdateCaptor = ArgumentCaptor.forClass(null); - verify(edsResourceWatcher).onChanged(edsUpdateCaptor.capture()); - EdsUpdate edsUpdate = edsUpdateCaptor.getValue(); - assertThat(edsUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(edsUpdate.getDropPolicies()) - .containsExactly( - new DropOverload("lb", 200), - new DropOverload("throttle", 1000)); - assertThat(edsUpdate.getLocalityLbEndpointsMap()) - .containsExactly( - new Locality("region1", "zone1", "subzone1"), - new LocalityLbEndpoints( - ImmutableList.of( - new LbEndpoint("192.168.0.1", 8080, - 2, true)), 1, 0), - new Locality("region3", "zone3", "subzone3"), - new LocalityLbEndpoints(ImmutableList.of(), 2, 1)); - - clusterLoadAssignments = ImmutableList.of( - Any.pack(buildClusterLoadAssignmentV2("cluster-foo.googleapis.com", - // 0 locality - ImmutableList.of(), - ImmutableList.of()))); - response = - buildDiscoveryResponseV2( - "1", clusterLoadAssignments, XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"); - responseObserver.onNext(response); - - // Client sent an ACK EDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "1", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"))); - - verify(edsResourceWatcher, times(2)).onChanged(edsUpdateCaptor.capture()); - edsUpdate = edsUpdateCaptor.getValue(); - assertThat(edsUpdate.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(edsUpdate.getDropPolicies()).isEmpty(); - assertThat(edsUpdate.getLocalityLbEndpointsMap()).isEmpty(); - } - - @Test - public void multipleEdsWatchers() { - EdsResourceWatcher watcher1 = mock(EdsResourceWatcher.class); - EdsResourceWatcher watcher2 = mock(EdsResourceWatcher.class); - EdsResourceWatcher watcher3 = mock(EdsResourceWatcher.class); - xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher1); - xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher2); - xdsClient.watchEdsResource("cluster-bar.googleapis.com", watcher3); - - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - // Client sends an EDS request containing all clusters being watched to management server. - verify(requestObserver) - .onNext( - argThat( - new DiscoveryRequestMatcher("", - ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); - - assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(2); - - // Management server sends back an EDS response contains ClusterLoadAssignment for only one of - // requested cluster. - List clusterLoadAssignments = ImmutableList.of( - Any.pack(buildClusterLoadAssignmentV2("cluster-foo.googleapis.com", - ImmutableList.of( - buildLocalityLbEndpointsV2("region1", "zone1", "subzone1", - ImmutableList.of( - buildLbEndpointV2("192.168.0.1", 8080, HealthStatus.HEALTHY, 2)), - 1, 0)), - ImmutableList.of()))); - - DiscoveryResponse response = - buildDiscoveryResponseV2("0", clusterLoadAssignments, - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"); - responseObserver.onNext(response); - - assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); - - // Client sent an ACK EDS request. - verify(requestObserver) - .onNext( - argThat( - new DiscoveryRequestMatcher("0", - ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"))); - - // Two watchers get notification of endpoint update for the cluster they are interested in. - ArgumentCaptor edsUpdateCaptor1 = ArgumentCaptor.forClass(null); - verify(watcher1).onChanged(edsUpdateCaptor1.capture()); - EdsUpdate edsUpdate1 = edsUpdateCaptor1.getValue(); - assertThat(edsUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(edsUpdate1.getLocalityLbEndpointsMap()) - .containsExactly( - new Locality("region1", "zone1", "subzone1"), - new LocalityLbEndpoints( - ImmutableList.of( - new LbEndpoint("192.168.0.1", 8080, - 2, true)), 1, 0)); - - ArgumentCaptor edsUpdateCaptor2 = ArgumentCaptor.forClass(null); - verify(watcher1).onChanged(edsUpdateCaptor2.capture()); - EdsUpdate edsUpdate2 = edsUpdateCaptor2.getValue(); - assertThat(edsUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(edsUpdate2.getLocalityLbEndpointsMap()) - .containsExactly( - new Locality("region1", "zone1", "subzone1"), - new LocalityLbEndpoints( - ImmutableList.of( - new LbEndpoint("192.168.0.1", 8080, - 2, true)), 1, 0)); - - verifyNoInteractions(watcher3); - - // Management server sends back another EDS response contains ClusterLoadAssignment for the - // other requested cluster. - clusterLoadAssignments = ImmutableList.of( - Any.pack(buildClusterLoadAssignmentV2("cluster-bar.googleapis.com", - ImmutableList.of( - buildLocalityLbEndpointsV2("region2", "zone2", "subzone2", - ImmutableList.of( - buildLbEndpointV2("192.168.234.52", 8888, HealthStatus.UNKNOWN, 5)), - 6, 0)), - ImmutableList.of()))); - - response = buildDiscoveryResponseV2("1", clusterLoadAssignments, - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"); - responseObserver.onNext(response); - - // Client sent an ACK EDS request. - verify(requestObserver) - .onNext( - argThat( - new DiscoveryRequestMatcher("1", - ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"))); - - // The corresponding watcher gets notified. - ArgumentCaptor edsUpdateCaptor3 = ArgumentCaptor.forClass(null); - verify(watcher3).onChanged(edsUpdateCaptor3.capture()); - EdsUpdate edsUpdate3 = edsUpdateCaptor3.getValue(); - assertThat(edsUpdate3.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); - assertThat(edsUpdate3.getLocalityLbEndpointsMap()) - .containsExactly( - new Locality("region2", "zone2", "subzone2"), - new LocalityLbEndpoints( - ImmutableList.of( - new LbEndpoint("192.168.234.52", 8888, - 5, true)), 6, 0)); - } - - /** - * (EDS response caching behavior) An endpoint watcher is registered for a cluster that already - * has some other endpoint watchers watching on. Endpoint information received previously is - * in local cache and notified to the new watcher immediately. - */ - @Test - public void watchEndpointsForClusterAlreadyBeingWatched() { - EdsResourceWatcher watcher1 = mock(EdsResourceWatcher.class); - xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher1); - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - // Client sends first EDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); - - assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); - - // Management server sends back an EDS response containing ClusterLoadAssignments for - // some cluster not requested. - List clusterLoadAssignments = ImmutableList.of( - Any.pack(buildClusterLoadAssignmentV2("cluster-foo.googleapis.com", - ImmutableList.of( - buildLocalityLbEndpointsV2("region1", "zone1", "subzone1", - ImmutableList.of( - buildLbEndpointV2("192.168.0.1", 8080, HealthStatus.HEALTHY, 2)), - 1, 0)), - ImmutableList.of()))); - - DiscoveryResponse response = - buildDiscoveryResponseV2("0", clusterLoadAssignments, - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"); - responseObserver.onNext(response); - - assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); - - // Client sent an ACK EDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"))); - - ArgumentCaptor edsUpdateCaptor1 = ArgumentCaptor.forClass(null); - verify(watcher1).onChanged(edsUpdateCaptor1.capture()); - EdsUpdate edsUpdate1 = edsUpdateCaptor1.getValue(); - assertThat(edsUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(edsUpdate1.getDropPolicies()).isEmpty(); - assertThat(edsUpdate1.getLocalityLbEndpointsMap()) - .containsExactly( - new Locality("region1", "zone1", "subzone1"), - new LocalityLbEndpoints( - ImmutableList.of( - new LbEndpoint("192.168.0.1", 8080, - 2, true)), 1, 0)); - - // A second endpoint watcher is registered for endpoints in the same cluster. - EdsResourceWatcher watcher2 = mock(EdsResourceWatcher.class); - xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher2); - - // Cached endpoint information is notified to the new watcher immediately, without sending - // another EDS request. - ArgumentCaptor edsUpdateCaptor2 = ArgumentCaptor.forClass(null); - verify(watcher2).onChanged(edsUpdateCaptor2.capture()); - EdsUpdate edsUpdate2 = edsUpdateCaptor2.getValue(); - assertThat(edsUpdate2.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(edsUpdate2.getDropPolicies()).isEmpty(); - assertThat(edsUpdate2.getLocalityLbEndpointsMap()) - .containsExactly( - new Locality("region1", "zone1", "subzone1"), - new LocalityLbEndpoints( - ImmutableList.of( - new LbEndpoint("192.168.0.1", 8080, - 2, true)), 1, 0)); - - verifyNoMoreInteractions(requestObserver); - assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); - } - - /** - * Basic operations of adding/canceling endpoint data watchers. - */ - @Test - public void addRemoveEdsWatchers() { - EdsResourceWatcher watcher1 = mock(EdsResourceWatcher.class); - xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher1); - - // Streaming RPC starts after a first watcher is added. - StreamObserver responseObserver = responseObservers.poll(); - StreamObserver requestObserver = requestObservers.poll(); - - // Client sends an EDS request to management server. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); - - // Management server sends back an EDS response with ClusterLoadAssignment for the requested - // cluster. - List clusterLoadAssignments = ImmutableList.of( - Any.pack(buildClusterLoadAssignmentV2("cluster-foo.googleapis.com", - ImmutableList.of( - buildLocalityLbEndpointsV2("region1", "zone1", "subzone1", - ImmutableList.of( - buildLbEndpointV2("192.168.0.1", 8080, HealthStatus.HEALTHY, 2), - buildLbEndpointV2("192.132.53.5", 80, HealthStatus.UNHEALTHY, 5)), - 1, 0)), - ImmutableList.of()))); - - DiscoveryResponse response = - buildDiscoveryResponseV2("0", clusterLoadAssignments, - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sent an ACK EDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"))); - - ArgumentCaptor edsUpdateCaptor1 = ArgumentCaptor.forClass(null); - verify(watcher1).onChanged(edsUpdateCaptor1.capture()); - EdsUpdate edsUpdate1 = edsUpdateCaptor1.getValue(); - assertThat(edsUpdate1.getClusterName()).isEqualTo("cluster-foo.googleapis.com"); - assertThat(edsUpdate1.getLocalityLbEndpointsMap()) - .containsExactly( - new Locality("region1", "zone1", "subzone1"), - new LocalityLbEndpoints( - ImmutableList.of( - new LbEndpoint("192.168.0.1", 8080, 2, true), - new LbEndpoint("192.132.53.5", 80,5, false)), - 1, 0)); - - // Add another endpoint watcher for a different cluster. - EdsResourceWatcher watcher2 = mock(EdsResourceWatcher.class); - xdsClient.watchEdsResource("cluster-bar.googleapis.com", watcher2); - - // Client sent a new EDS request for all interested resources. - verify(requestObserver) - .onNext( - argThat( - new DiscoveryRequestMatcher("0", - ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"))); - - // Management server sends back an EDS response with ClusterLoadAssignment for one of requested - // cluster. - clusterLoadAssignments = ImmutableList.of( - Any.pack(buildClusterLoadAssignmentV2("cluster-bar.googleapis.com", - ImmutableList.of( - buildLocalityLbEndpointsV2("region2", "zone2", "subzone2", - ImmutableList.of( - buildLbEndpointV2("192.168.312.6", 443, HealthStatus.HEALTHY, 1)), - 6, 0)), - ImmutableList.of()))); - - response = buildDiscoveryResponseV2("1", clusterLoadAssignments, - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"); - responseObserver.onNext(response); - - // Client sent an ACK EDS request for all interested resources. - verify(requestObserver) - .onNext( - argThat( - new DiscoveryRequestMatcher("1", - ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"))); - - ArgumentCaptor edsUpdateCaptor2 = ArgumentCaptor.forClass(null); - verify(watcher2).onChanged(edsUpdateCaptor2.capture()); - EdsUpdate edsUpdate2 = edsUpdateCaptor2.getValue(); - assertThat(edsUpdate2.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); - assertThat(edsUpdate2.getLocalityLbEndpointsMap()) - .containsExactly( - new Locality("region2", "zone2", "subzone2"), - new LocalityLbEndpoints( - ImmutableList.of( - new LbEndpoint("192.168.312.6", 443, 1, true)), - 6, 0)); - - // Cancel one of the watcher. - xdsClient.cancelEdsResourceWatch("cluster-foo.googleapis.com", watcher1); - - // Since the cancelled watcher was the last watcher interested in that cluster, client - // sent an new EDS request to unsubscribe from that cluster. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "1", "cluster-bar.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"))); - - // Management server should not respond as it had previously sent the requested resource. - - // Cancel the other watcher. - xdsClient.cancelEdsResourceWatch("cluster-bar.googleapis.com", watcher2); - - // Since the cancelled watcher was the last watcher interested in that cluster, client - // sent an new EDS request to unsubscribe from that cluster. - verify(requestObserver) - .onNext( - argThat( - new DiscoveryRequestMatcher("1", - ImmutableList.of(), // empty resources - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0001"))); - - // All endpoint watchers have been cancelled. - - // Management server sends back an EDS response for updating previously sent resources. - clusterLoadAssignments = ImmutableList.of( - Any.pack(buildClusterLoadAssignmentV2("cluster-foo.googleapis.com", - ImmutableList.of( - buildLocalityLbEndpointsV2("region3", "zone3", "subzone3", - ImmutableList.of( - buildLbEndpointV2("192.168.432.6", 80, HealthStatus.HEALTHY, 2)), - 3, 0)), - ImmutableList.of())), - Any.pack(buildClusterLoadAssignmentV2("cluster-bar.googleapis.com", - ImmutableList.of( - buildLocalityLbEndpointsV2("region4", "zone4", "subzone4", - ImmutableList.of( - buildLbEndpointV2("192.168.75.6", 8888, HealthStatus.HEALTHY, 2)), - 3, 0)), - ImmutableList.of()))); - - response = buildDiscoveryResponseV2("2", clusterLoadAssignments, - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0002"); - responseObserver.onNext(response); - - // Client sent an ACK EDS request. - verify(requestObserver) - .onNext( - argThat( - new DiscoveryRequestMatcher("2", - ImmutableList.of(), // empty resources - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0002"))); - - // Cancelled watchers do not receive notification. - verifyNoMoreInteractions(watcher1, watcher2); - - // A new endpoint watcher is added to watch an old but was no longer interested in cluster. - EdsResourceWatcher watcher3 = mock(EdsResourceWatcher.class); - xdsClient.watchEdsResource("cluster-bar.googleapis.com", watcher3); - - // Nothing should be notified to the new watcher as we are still waiting management server's - // latest response. - // Cached endpoint data should have been purged. - verify(watcher3, never()).onChanged(any(EdsUpdate.class)); - - // An EDS request is sent to re-subscribe the cluster again. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "2", "cluster-bar.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0002"))); - - // Management server sends back an EDS response for re-subscribed resource. - clusterLoadAssignments = ImmutableList.of( - Any.pack(buildClusterLoadAssignmentV2("cluster-bar.googleapis.com", - ImmutableList.of( - buildLocalityLbEndpointsV2("region4", "zone4", "subzone4", - ImmutableList.of( - buildLbEndpointV2("192.168.75.6", 8888, HealthStatus.HEALTHY, 2)), - 3, 0)), - ImmutableList.of()))); - - response = buildDiscoveryResponseV2("3", clusterLoadAssignments, - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0003"); - responseObserver.onNext(response); - - ArgumentCaptor edsUpdateCaptor3 = ArgumentCaptor.forClass(null); - verify(watcher3).onChanged(edsUpdateCaptor3.capture()); - EdsUpdate edsUpdate3 = edsUpdateCaptor3.getValue(); - assertThat(edsUpdate3.getClusterName()).isEqualTo("cluster-bar.googleapis.com"); - assertThat(edsUpdate3.getLocalityLbEndpointsMap()) - .containsExactly( - new Locality("region4", "zone4", "subzone4"), - new LocalityLbEndpoints( - ImmutableList.of( - new LbEndpoint("192.168.75.6", 8888, 2, true)), - 3, 0)); - - // Client sent an ACK EDS request. - verify(requestObserver) - .onNext( - argThat( - new DiscoveryRequestMatcher("3", - ImmutableList.of("cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0003"))); - } - - @Test - public void addRemoveEdsWatcherWhileInitialResourceFetchInProgress() { - EdsResourceWatcher watcher1 = mock(EdsResourceWatcher.class); - xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher1); - - // Streaming RPC starts after a first watcher is added. - StreamObserver requestObserver = requestObservers.poll(); - - // Client sends an EDS request to management server. - verify(requestObserver) - .onNext( - argThat( - new DiscoveryRequestMatcher("", "cluster-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); - assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); - - fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC - 1, TimeUnit.SECONDS); - - EdsResourceWatcher watcher2 = mock(EdsResourceWatcher.class); - EdsResourceWatcher watcher3 = mock(EdsResourceWatcher.class); - EdsResourceWatcher watcher4 = mock(EdsResourceWatcher.class); - xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher2); - xdsClient.watchEdsResource("cluster-bar.googleapis.com", watcher3); - xdsClient.watchEdsResource("cluster-bar.googleapis.com", watcher4); - - // Client sends a new EDS request for updating the latest resource subscription. - verify(requestObserver) - .onNext( - argThat( - new DiscoveryRequestMatcher("", - ImmutableList.of("cluster-foo.googleapis.com", "cluster-bar.googleapis.com"), - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); - assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(2); - - fakeClock.forwardTime(1, TimeUnit.SECONDS); - assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); - - // EDS resource "cluster-foo.googleapis.com" is known to be absent. - verify(watcher1).onResourceDoesNotExist("cluster-foo.googleapis.com"); - verify(watcher2).onResourceDoesNotExist("cluster-foo.googleapis.com"); - - // The absence result is known immediately. - EdsResourceWatcher watcher5 = mock(EdsResourceWatcher.class); - xdsClient.watchEdsResource("cluster-foo.googleapis.com", watcher5); - verify(watcher5).onResourceDoesNotExist("cluster-foo.googleapis.com"); - - assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); - ScheduledTask timeoutTask = Iterables.getOnlyElement(fakeClock.getPendingTasks()); - - // Cancel watchers while discovery for resource "cluster-bar.googleapis.com" is still - // in progress. - xdsClient.cancelEdsResourceWatch("cluster-bar.googleapis.com", watcher3); - assertThat(timeoutTask.isCancelled()).isFalse(); - xdsClient.cancelEdsResourceWatch("cluster-bar.googleapis.com", watcher4); - - // Client sends an EDS request for resource subscription update (Omitted). - - fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); - - assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); - assertThat(timeoutTask.isCancelled()).isTrue(); - - verifyNoInteractions(watcher3, watcher4); - } - - @Test - public void cdsUpdateForEdsServiceNameChange() { - xdsClient.watchCdsResource("cluster-foo.googleapis.com", cdsResourceWatcher); - StreamObserver responseObserver = responseObservers.poll(); - - // Management server sends back a CDS response containing requested resource. - List clusters = ImmutableList.of( - Any.pack(buildClusterV2("cluster-foo.googleapis.com", "cluster-foo:service-bar", false))); - DiscoveryResponse response = - buildDiscoveryResponseV2("0", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"); - responseObserver.onNext(response); - - xdsClient.watchEdsResource("cluster-foo:service-bar", edsResourceWatcher); - - // Management server sends back an EDS response for resource "cluster-foo:service-bar". - List clusterLoadAssignments = ImmutableList.of( - Any.pack(buildClusterLoadAssignmentV2("cluster-foo:service-bar", - ImmutableList.of( - buildLocalityLbEndpointsV2("region1", "zone1", "subzone1", - ImmutableList.of( - buildLbEndpointV2("192.168.0.1", 8080, HealthStatus.HEALTHY, 2)), - 1, 0)), - ImmutableList.of()))); - response = - buildDiscoveryResponseV2("0", clusterLoadAssignments, - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "0000"); - responseObserver.onNext(response); - - ArgumentCaptor edsUpdateCaptor = ArgumentCaptor.forClass(null); - verify(edsResourceWatcher).onChanged(edsUpdateCaptor.capture()); - EdsUpdate edsUpdate = edsUpdateCaptor.getValue(); - assertThat(edsUpdate.getClusterName()).isEqualTo("cluster-foo:service-bar"); - assertThat(edsUpdate.getDropPolicies()).isEmpty(); - assertThat(edsUpdate.getLocalityLbEndpointsMap()) - .containsExactly( - new Locality("region1", "zone1", "subzone1"), - new LocalityLbEndpoints( - ImmutableList.of( - new LbEndpoint("192.168.0.1", 8080, - 2, true)), 1, 0)); - - // Management server sends another CDS response for removing cluster service - // "cluster-foo:service-blade" with replacement of "cluster-foo:service-blade". - clusters = ImmutableList.of( - Any.pack(buildClusterV2("cluster-foo.googleapis.com", "cluster-foo:service-blade", false))); - response = - buildDiscoveryResponseV2("1", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0001"); - responseObserver.onNext(response); - - // Watcher get notification for endpoint resource "cluster-foo:service-bar" being deleted. - verify(edsResourceWatcher).onResourceDoesNotExist("cluster-foo:service-bar"); - } - - /** - * RPC stream closed and retry during the period of first time resolving service config - * (LDS/RDS only). - */ - @Test - public void streamClosedAndRetryWhenResolvingConfig() { - InOrder inOrder = - Mockito.inOrder(mockedDiscoveryService, backoffPolicyProvider, backoffPolicy1, - backoffPolicy2); - xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); - - ArgumentCaptor> responseObserverCaptor = - ArgumentCaptor.forClass(null); - inOrder.verify(mockedDiscoveryService) - .streamAggregatedResources(responseObserverCaptor.capture()); - StreamObserver responseObserver = - responseObserverCaptor.getValue(); // same as responseObservers.poll() - StreamObserver requestObserver = requestObservers.poll(); - - // Client sends an LDS request for the host name (with port) to management server. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - - // Management server closes the RPC stream immediately. - responseObserver.onCompleted(); - inOrder.verify(backoffPolicyProvider).get(); - inOrder.verify(backoffPolicy1).nextBackoffNanos(); - assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); - - // Retry after backoff. - fakeClock.forwardNanos(9L); - assertThat(requestObservers).isEmpty(); - fakeClock.forwardNanos(1L); - inOrder.verify(mockedDiscoveryService) - .streamAggregatedResources(responseObserverCaptor.capture()); - responseObserver = responseObserverCaptor.getValue(); - requestObserver = requestObservers.poll(); - - // Client retried by sending an LDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - - // Management server closes the RPC stream with an error. - responseObserver.onError(Status.UNAVAILABLE.asException()); - verifyNoMoreInteractions(backoffPolicyProvider); - inOrder.verify(backoffPolicy1).nextBackoffNanos(); - assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); - - // Retry after backoff. - fakeClock.forwardNanos(99L); - assertThat(requestObservers).isEmpty(); - fakeClock.forwardNanos(1L); - inOrder.verify(mockedDiscoveryService) - .streamAggregatedResources(responseObserverCaptor.capture()); - responseObserver = responseObserverCaptor.getValue(); - requestObserver = requestObservers.poll(); - - // Client retried again by sending an LDS. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - - // Management server responses with a listener for the requested resource. - Rds rdsConfig = - Rds.newBuilder() - .setConfigSource( - ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) - .setRouteConfigName("route-foo.googleapis.com") - .build(); - - List listeners = ImmutableList.of( - Any.pack(buildListenerV2(TARGET_AUTHORITY, /* matching resource */ - Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) - ); - DiscoveryResponse ldsResponse = - buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); - responseObserver.onNext(ldsResponse); - - // Client sent back an ACK LDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"))); - - // Client sent an RDS request based on the received listener. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "route-foo.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_RDS_V2, ""))); - - // Management server encounters an error and closes the stream. - responseObserver.onError(Status.UNKNOWN.asException()); - - // Reset backoff and retry immediately. - inOrder.verify(backoffPolicyProvider).get(); - fakeClock.runDueTasks(); - inOrder.verify(mockedDiscoveryService) - .streamAggregatedResources(responseObserverCaptor.capture()); - responseObserver = responseObserverCaptor.getValue(); - requestObserver = requestObservers.poll(); - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - - // RPC stream closed immediately - responseObserver.onError(Status.UNKNOWN.asException()); - inOrder.verify(backoffPolicy2).nextBackoffNanos(); - assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); - - // Retry after backoff. - fakeClock.forwardNanos(19L); - assertThat(requestObservers).isEmpty(); - fakeClock.forwardNanos(1L); - inOrder.verify(mockedDiscoveryService) - .streamAggregatedResources(responseObserverCaptor.capture()); - responseObserver = responseObserverCaptor.getValue(); - requestObserver = requestObservers.poll(); - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - - // Management server sends an LDS response. - ldsResponse = - buildDiscoveryResponseV2("1", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0001"); - responseObserver.onNext(ldsResponse); - - // Client sends an ACK LDS request and an RDS request for "route-foo.googleapis.com". (Omitted) - - List routeConfigs = ImmutableList.of( - Any.pack( - buildRouteConfigurationV2( - "route-foo.googleapis.com", // target route configuration - ImmutableList.of( - buildVirtualHostV2( - ImmutableList.of(TARGET_AUTHORITY), // matching virtual host - "cluster.googleapis.com"))))); - DiscoveryResponse rdsResponse = - buildDiscoveryResponseV2("0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); - // Management server sends an RDS response. - responseObserver.onNext(rdsResponse); - - // Client has resolved the cluster based on the RDS response. - ArgumentCaptor configUpdateCaptor = ArgumentCaptor.forClass(null); - verify(configWatcher).onConfigChanged(configUpdateCaptor.capture()); - assertConfigUpdateContainsSingleClusterRoute( - configUpdateCaptor.getValue(), "cluster.googleapis.com"); - - // RPC stream closed with an error again. - responseObserver.onError(Status.UNKNOWN.asException()); - - // Reset backoff and retry immediately. - inOrder.verify(backoffPolicyProvider).get(); - fakeClock.runDueTasks(); - requestObserver = requestObservers.poll(); - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "1", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - - verifyNoMoreInteractions(backoffPolicyProvider, backoffPolicy1, backoffPolicy2); - } - - /** - * RPC stream close and retry while there are config/cluster/endpoint watchers registered. - */ - @Test - public void streamClosedAndRetry() { - InOrder inOrder = - Mockito.inOrder(mockedDiscoveryService, backoffPolicyProvider, backoffPolicy1, - backoffPolicy2); - xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); - - ArgumentCaptor> responseObserverCaptor = - ArgumentCaptor.forClass(null); - inOrder.verify(mockedDiscoveryService) - .streamAggregatedResources(responseObserverCaptor.capture()); - StreamObserver responseObserver = - responseObserverCaptor.getValue(); // same as responseObservers.poll() - StreamObserver requestObserver = requestObservers.poll(); - - waitUntilConfigResolved(responseObserver); - ArgumentCaptor statusCaptor = ArgumentCaptor.forClass(null); - - // Start watching cluster information. - xdsClient.watchCdsResource("cluster.googleapis.com", cdsResourceWatcher); - - // Client sent first CDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); - - // Start watching endpoint information. - xdsClient.watchEdsResource("cluster.googleapis.com", edsResourceWatcher); - - // Client sent first EDS request. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); - - // Management server closes the RPC stream with an error. - responseObserver.onError(Status.UNKNOWN.asException()); - verify(configWatcher).onError(statusCaptor.capture()); - assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNKNOWN); - verify(cdsResourceWatcher).onError(statusCaptor.capture()); - assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNKNOWN); - verify(edsResourceWatcher).onError(statusCaptor.capture()); - assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNKNOWN); - - // Resets backoff and retry immediately. - inOrder.verify(backoffPolicyProvider).get(); - fakeClock.runDueTasks(); - inOrder.verify(mockedDiscoveryService) - .streamAggregatedResources(responseObserverCaptor.capture()); - responseObserver = responseObserverCaptor.getValue(); - requestObserver = requestObservers.poll(); - - // Retry resumes requests for all wanted resources. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); - - // Management server becomes unreachable. - responseObserver.onError(Status.UNAVAILABLE.asException()); - verify(configWatcher, times(2)).onError(statusCaptor.capture()); - assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); - verify(cdsResourceWatcher, times(2)).onError(statusCaptor.capture()); - assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); - verify(edsResourceWatcher, times(2)).onError(statusCaptor.capture()); - assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); - inOrder.verify(backoffPolicy1).nextBackoffNanos(); - assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); - - // Retry after backoff. - fakeClock.forwardNanos(9L); - assertThat(requestObservers).isEmpty(); - fakeClock.forwardNanos(1L); - inOrder.verify(mockedDiscoveryService) - .streamAggregatedResources(responseObserverCaptor.capture()); - responseObserver = responseObserverCaptor.getValue(); - requestObserver = requestObservers.poll(); - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); - - // Management server is still not reachable. - responseObserver.onError(Status.UNAVAILABLE.asException()); - verify(configWatcher, times(3)).onError(statusCaptor.capture()); - assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); - verify(cdsResourceWatcher, times(3)).onError(statusCaptor.capture()); - assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); - verify(edsResourceWatcher, times(3)).onError(statusCaptor.capture()); - assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); - inOrder.verify(backoffPolicy1).nextBackoffNanos(); - assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); - - // Retry after backoff. - fakeClock.forwardNanos(99L); - assertThat(requestObservers).isEmpty(); - fakeClock.forwardNanos(1L); - inOrder.verify(mockedDiscoveryService) - .streamAggregatedResources(responseObserverCaptor.capture()); - responseObserver = responseObserverCaptor.getValue(); - requestObserver = requestObservers.poll(); - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); - - // Management server sends back a CDS response. - List clusters = ImmutableList.of( - Any.pack(buildClusterV2("cluster.googleapis.com", null, false))); - DiscoveryResponse cdsResponse = - buildDiscoveryResponseV2("0", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"); - responseObserver.onNext(cdsResponse); - - // Client sent an CDS ACK request (Omitted). - - // Management server closes the RPC stream. - responseObserver.onCompleted(); - verify(configWatcher, times(4)).onError(any(Status.class)); - verify(cdsResourceWatcher, times(4)).onError(any(Status.class)); - verify(edsResourceWatcher, times(4)).onError(any(Status.class)); - - // Resets backoff and retry immediately - inOrder.verify(backoffPolicyProvider).get(); - fakeClock.runDueTasks(); - inOrder.verify(mockedDiscoveryService) - .streamAggregatedResources(responseObserverCaptor.capture()); - responseObserver = responseObserverCaptor.getValue(); - requestObserver = requestObservers.poll(); - - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); - - // Management server becomes unreachable again. - responseObserver.onError(Status.UNAVAILABLE.asException()); - verify(configWatcher, times(5)).onError(statusCaptor.capture()); - assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); - verify(cdsResourceWatcher, times(5)).onError(statusCaptor.capture()); - assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); - verify(edsResourceWatcher, times(5)).onError(statusCaptor.capture()); - assertThat(statusCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); - inOrder.verify(backoffPolicy2).nextBackoffNanos(); - assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); - - // Retry after backoff. - fakeClock.forwardNanos(19L); - assertThat(requestObservers).isEmpty(); - fakeClock.forwardNanos(1L); - inOrder.verify(mockedDiscoveryService) - .streamAggregatedResources(responseObserverCaptor.capture()); - requestObserver = requestObservers.poll(); - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); - - verifyNoMoreInteractions(mockedDiscoveryService, backoffPolicyProvider, backoffPolicy1, - backoffPolicy2); - } - - /** - * RPC stream closed and retry while some cluster/endpoint watchers have changed (added/removed). - */ - @Test - public void streamClosedAndRetryRaceWithAddingAndRemovingWatchers() { - InOrder inOrder = - Mockito.inOrder(mockedDiscoveryService, backoffPolicyProvider, backoffPolicy1, - backoffPolicy2); - xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); - - ArgumentCaptor> responseObserverCaptor = - ArgumentCaptor.forClass(null); - inOrder.verify(mockedDiscoveryService) - .streamAggregatedResources(responseObserverCaptor.capture()); - StreamObserver responseObserver = - responseObserverCaptor.getValue(); // same as responseObservers.poll() - requestObservers.poll(); - - waitUntilConfigResolved(responseObserver); - - // Management server closes RPC stream. - responseObserver.onCompleted(); - - // Resets backoff and retry immediately. - inOrder.verify(backoffPolicyProvider).get(); - fakeClock.runDueTasks(); - inOrder.verify(mockedDiscoveryService) - .streamAggregatedResources(responseObserverCaptor.capture()); - responseObserver = responseObserverCaptor.getValue(); - StreamObserver requestObserver = requestObservers.poll(); - - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - - // Management server becomes unreachable. - responseObserver.onError(Status.UNAVAILABLE.asException()); - inOrder.verify(backoffPolicy1).nextBackoffNanos(); - assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); - - // Start watching cluster information while RPC stream is still in retry backoff. - xdsClient.watchCdsResource("cluster.googleapis.com", cdsResourceWatcher); - - // Retry after backoff. - fakeClock.forwardNanos(9L); - assertThat(requestObservers).isEmpty(); - fakeClock.forwardNanos(1L); - inOrder.verify(mockedDiscoveryService) - .streamAggregatedResources(responseObserverCaptor.capture()); - responseObserver = responseObserverCaptor.getValue(); - requestObserver = requestObservers.poll(); - - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); - - // Management server is still unreachable. - responseObserver.onError(Status.UNAVAILABLE.asException()); - inOrder.verify(backoffPolicy1).nextBackoffNanos(); - assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); - - // Start watching endpoint information while RPC stream is still in retry backoff. - xdsClient.watchEdsResource("cluster.googleapis.com", edsResourceWatcher); - - // Retry after backoff. - fakeClock.forwardNanos(99L); - assertThat(requestObservers).isEmpty(); - fakeClock.forwardNanos(1L); - inOrder.verify(mockedDiscoveryService) - .streamAggregatedResources(responseObserverCaptor.capture()); - responseObserver = responseObserverCaptor.getValue(); - requestObserver = requestObservers.poll(); - - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); - - // Management server sends back a CDS response. - List clusters = ImmutableList.of( - Any.pack(buildClusterV2("cluster.googleapis.com", null, false))); - DiscoveryResponse cdsResponse = - buildDiscoveryResponseV2("0", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "0000"); - responseObserver.onNext(cdsResponse); - - // Client sent an CDS ACK request (Omitted). - - // No longer interested in endpoint information after RPC resumes. - xdsClient.cancelEdsResourceWatch("cluster.googleapis.com", edsResourceWatcher); - // Client updates EDS resource subscription immediately. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", ImmutableList.of(), - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); - - // Become interested in endpoints of another cluster. - xdsClient.watchEdsResource("cluster2.googleapis.com", edsResourceWatcher); - // Client updates EDS resource subscription immediately. - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster2.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); - - // Management server closes the RPC stream again. - responseObserver.onCompleted(); - - // Resets backoff and retry immediately. - inOrder.verify(backoffPolicyProvider).get(); - fakeClock.runDueTasks(); - inOrder.verify(mockedDiscoveryService) - .streamAggregatedResources(responseObserverCaptor.capture()); - responseObserver = responseObserverCaptor.getValue(); - requestObserver = requestObservers.poll(); - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster2.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); - - // Management server becomes unreachable again. - responseObserver.onError(Status.UNAVAILABLE.asException()); - inOrder.verify(backoffPolicy2).nextBackoffNanos(); - assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); - - // No longer interested in previous cluster and endpoints in that cluster. - xdsClient.cancelCdsResourceWatch("cluster.googleapis.com", cdsResourceWatcher); - xdsClient.cancelEdsResourceWatch("cluster2.googleapis.com", edsResourceWatcher); - - // Retry after backoff. - fakeClock.forwardNanos(19L); - assertThat(requestObservers).isEmpty(); - fakeClock.forwardNanos(1L); - inOrder.verify(mockedDiscoveryService) - .streamAggregatedResources(responseObserverCaptor.capture()); - requestObserver = requestObservers.poll(); - - verify(requestObserver) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - verify(requestObserver, never()) - .onNext(eq(buildDiscoveryRequestV2(NODE, "0", "cluster.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_CDS_V2, ""))); - verify(requestObserver, never()) - .onNext(eq(buildDiscoveryRequestV2(NODE, "", "cluster2.googleapis.com", - XdsClientImpl.ADS_TYPE_URL_EDS_V2, ""))); - - verifyNoMoreInteractions(mockedDiscoveryService, backoffPolicyProvider, backoffPolicy1, - backoffPolicy2); - } - - @Test - public void streamClosedAndRetryReschedulesAllResourceFetchTimer() { - InOrder inOrder = - Mockito.inOrder(mockedDiscoveryService, backoffPolicyProvider, backoffPolicy1, - backoffPolicy2); - xdsClient.watchConfigData(TARGET_AUTHORITY, configWatcher); - - ArgumentCaptor> responseObserverCaptor = - ArgumentCaptor.forClass(null); - inOrder.verify(mockedDiscoveryService) - .streamAggregatedResources(responseObserverCaptor.capture()); - StreamObserver responseObserver = - responseObserverCaptor.getValue(); // same as responseObservers.poll() - - // Management server sends back an LDS response telling client to do RDS. - Rds rdsConfig = - Rds.newBuilder() - // Must set to use ADS. - .setConfigSource( - ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) - .setRouteConfigName("route-foo.googleapis.com") - .build(); - - List listeners = ImmutableList.of( - Any.pack(buildListenerV2(TARGET_AUTHORITY, /* matching resource */ - Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) - ); - DiscoveryResponse response = - buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); - responseObserver.onNext(response); - - // Client sent an RDS request for resource "route-foo.googleapis.com" (Omitted). - - ScheduledTask rdsRespTimer = - Iterables.getOnlyElement( - fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); - assertThat(rdsRespTimer.isCancelled()).isFalse(); - fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC - 1, TimeUnit.SECONDS); - - // RPC stream is broken while the initial fetch for the resource is not complete. - responseObserver.onError(Status.UNAVAILABLE.asException()); - assertThat(rdsRespTimer.isCancelled()).isTrue(); - - // Reset backoff and retry immediately. - inOrder.verify(backoffPolicyProvider).get(); - fakeClock.runDueTasks(); - inOrder.verify(mockedDiscoveryService) - .streamAggregatedResources(responseObserverCaptor.capture()); - responseObserver = responseObserverCaptor.getValue(); - StreamObserver requestObserver = requestObservers.poll(); - - ScheduledTask ldsRespTimer = - Iterables.getOnlyElement( - fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); - assertThat(ldsRespTimer.getDelay(TimeUnit.SECONDS)) - .isEqualTo(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC); - - // Client resumed requests and management server sends back LDS resources again. - verify(requestObserver).onNext( - eq(buildDiscoveryRequestV2(NODE, "", TARGET_AUTHORITY, - XdsClientImpl.ADS_TYPE_URL_LDS_V2, ""))); - responseObserver.onNext(response); - - // Client sent an RDS request for resource "route-foo.googleapis.com" (Omitted). - - assertThat(ldsRespTimer.isCancelled()).isTrue(); - rdsRespTimer = - Iterables.getOnlyElement( - fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); - assertThat(rdsRespTimer.getDelay(TimeUnit.SECONDS)) - .isEqualTo(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC); - - // Management server sends back an RDS response containing the RouteConfiguration - // for the requested resource. - List routeConfigs = ImmutableList.of( - Any.pack( - buildRouteConfigurationV2( - "route-foo.googleapis.com", // target route configuration - ImmutableList.of( - buildVirtualHostV2( - ImmutableList.of(TARGET_AUTHORITY), // matching virtual host - "cluster-foo.googleapis.com"))))); - response = buildDiscoveryResponseV2( - "0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); - responseObserver.onNext(response); - - assertThat(rdsRespTimer.isCancelled()).isTrue(); - - // Resets RPC stream again. - responseObserver.onError(Status.UNAVAILABLE.asException()); - // Reset backoff and retry immediately. - inOrder.verify(backoffPolicyProvider).get(); - fakeClock.runDueTasks(); - inOrder.verify(mockedDiscoveryService) - .streamAggregatedResources(responseObserverCaptor.capture()); - responseObserver = responseObserverCaptor.getValue(); - - // Client/server resumed LDS/RDS request/response (Omitted). - - // Start watching cluster data. - xdsClient.watchCdsResource("cluster-foo.googleapis.com", cdsResourceWatcher); - ScheduledTask cdsRespTimeoutTask = - Iterables.getOnlyElement( - fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); - assertThat(cdsRespTimeoutTask.isCancelled()).isFalse(); - fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC - 1, TimeUnit.SECONDS); - - // RPC stream is broken while the initial fetch for the resource is not complete. - responseObserver.onError(Status.UNAVAILABLE.asException()); - assertThat(cdsRespTimeoutTask.isCancelled()).isTrue(); - inOrder.verify(backoffPolicy2).nextBackoffNanos(); - assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); - - // Retry after backoff. - fakeClock.forwardNanos(20L); - inOrder.verify(mockedDiscoveryService) - .streamAggregatedResources(responseObserverCaptor.capture()); - responseObserver = responseObserverCaptor.getValue(); - - // Timer is rescheduled as the client restarts the resource fetch. - cdsRespTimeoutTask = - Iterables.getOnlyElement( - fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); - assertThat(cdsRespTimeoutTask.isCancelled()).isFalse(); - assertThat(cdsRespTimeoutTask.getDelay(TimeUnit.SECONDS)) - .isEqualTo(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC); - - // Start watching endpoint data. - xdsClient.watchEdsResource("cluster-foo.googleapis.com", edsResourceWatcher); - ScheduledTask edsTimeoutTask = - Iterables.getOnlyElement( - fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); - assertThat(edsTimeoutTask.getDelay(TimeUnit.SECONDS)) - .isEqualTo(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC); - - // RPC stream is broken again. - responseObserver.onError(Status.UNAVAILABLE.asException()); - - assertThat(edsTimeoutTask.isCancelled()).isTrue(); - inOrder.verify(backoffPolicy2).nextBackoffNanos(); - assertThat(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)).hasSize(1); - - fakeClock.forwardNanos(200L); - inOrder.verify(mockedDiscoveryService) - .streamAggregatedResources(responseObserverCaptor.capture()); - - assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); - assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); - } - - /** - * Tests sending a streaming LRS RPC for each cluster to report loads for. - */ - @Test - public void reportLoadStatsToServer() { - String clusterName = "cluster-foo.googleapis.com"; - ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(null); - xdsClient.addClientStats(clusterName, null); - xdsClient.reportClientStats(); - LoadReportCall lrsCall = loadReportCalls.poll(); - verify(lrsCall.requestObserver).onNext(requestCaptor.capture()); - assertThat(requestCaptor.getValue().getClusterStatsCount()) - .isEqualTo(0); // initial request - - lrsCall.responseObserver.onNext( - LoadStatsResponse.newBuilder() - .addClusters(clusterName) - .setLoadReportingInterval(Durations.fromNanos(1000L)) - .build()); - fakeClock.forwardNanos(1000L); - verify(lrsCall.requestObserver, times(2)).onNext(requestCaptor.capture()); - ClusterStats report = Iterables.getOnlyElement(requestCaptor.getValue().getClusterStatsList()); - assertThat(report.getClusterName()).isEqualTo(clusterName); - - xdsClient.removeClientStats(clusterName, null); - fakeClock.forwardNanos(1000L); - verify(lrsCall.requestObserver, times(3)).onNext(requestCaptor.capture()); - assertThat(requestCaptor.getValue().getClusterStatsCount()) - .isEqualTo(0); // no more stats reported - - xdsClient.cancelClientStatsReport(); - assertThat(lrsEnded.get()).isTrue(); - // See more test on LoadReportClientTest.java - } - - // Simulates the use case of watching clusters/endpoints based on service config resolved by - // LDS/RDS. - private void waitUntilConfigResolved(StreamObserver responseObserver) { - // Client sent an LDS request for resource TARGET_AUTHORITY (Omitted). - - // Management server responses with a listener telling client to do RDS. - Rds rdsConfig = - Rds.newBuilder() - .setConfigSource( - ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance())) - .setRouteConfigName("route-foo.googleapis.com") - .build(); - - List listeners = ImmutableList.of( - Any.pack(buildListenerV2(TARGET_AUTHORITY, /* matching resource */ - Any.pack(HttpConnectionManager.newBuilder().setRds(rdsConfig).build()))) - ); - DiscoveryResponse ldsResponse = - buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); - responseObserver.onNext(ldsResponse); - - // Client sent an LDS ACK request and an RDS request for resource - // "route-foo.googleapis.com" (Omitted). - - // Management server sends an RDS response. - List routeConfigs = ImmutableList.of( - Any.pack( - buildRouteConfigurationV2( - "route-foo.googleapis.com", // target route configuration - ImmutableList.of( - buildVirtualHostV2( - ImmutableList.of(TARGET_AUTHORITY), // matching virtual host - "cluster.googleapis.com"))))); - DiscoveryResponse rdsResponse = - buildDiscoveryResponseV2("0", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0000"); - responseObserver.onNext(rdsResponse); - } - - @Test - public void matchHostName_exactlyMatch() { - String pattern = "foo.googleapis.com"; - assertThat(XdsClientImpl.matchHostName("bar.googleapis.com", pattern)).isFalse(); - assertThat(XdsClientImpl.matchHostName("fo.googleapis.com", pattern)).isFalse(); - assertThat(XdsClientImpl.matchHostName("oo.googleapis.com", pattern)).isFalse(); - assertThat(XdsClientImpl.matchHostName("googleapis.com", pattern)).isFalse(); - assertThat(XdsClientImpl.matchHostName("foo.googleapis", pattern)).isFalse(); - assertThat(XdsClientImpl.matchHostName("foo.googleapis.com", pattern)).isTrue(); - } - - @Test - public void matchHostName_prefixWildcard() { - String pattern = "*.foo.googleapis.com"; - assertThat(XdsClientImpl.matchHostName("foo.googleapis.com", pattern)).isFalse(); - assertThat(XdsClientImpl.matchHostName("bar-baz.foo.googleapis", pattern)).isFalse(); - assertThat(XdsClientImpl.matchHostName("bar.foo.googleapis.com", pattern)).isTrue(); - pattern = "*-bar.foo.googleapis.com"; - assertThat(XdsClientImpl.matchHostName("bar.foo.googleapis.com", pattern)).isFalse(); - assertThat(XdsClientImpl.matchHostName("baz-bar.foo.googleapis", pattern)).isFalse(); - assertThat(XdsClientImpl.matchHostName("-bar.foo.googleapis.com", pattern)).isFalse(); - assertThat(XdsClientImpl.matchHostName("baz-bar.foo.googleapis.com", pattern)) - .isTrue(); - } - - @Test - public void matchHostName_postfixWildCard() { - String pattern = "foo.*"; - assertThat(XdsClientImpl.matchHostName("bar.googleapis.com", pattern)).isFalse(); - assertThat(XdsClientImpl.matchHostName("bar.foo.googleapis.com", pattern)).isFalse(); - assertThat(XdsClientImpl.matchHostName("foo.googleapis.com", pattern)).isTrue(); - assertThat(XdsClientImpl.matchHostName("foo.com", pattern)).isTrue(); - pattern = "foo-*"; - assertThat(XdsClientImpl.matchHostName("bar-.googleapis.com", pattern)).isFalse(); - assertThat(XdsClientImpl.matchHostName("foo.googleapis.com", pattern)).isFalse(); - assertThat(XdsClientImpl.matchHostName("foo.googleapis.com", pattern)).isFalse(); - assertThat(XdsClientImpl.matchHostName("foo-", pattern)).isFalse(); - assertThat(XdsClientImpl.matchHostName("foo-bar.com", pattern)).isTrue(); - assertThat(XdsClientImpl.matchHostName("foo-.com", pattern)).isTrue(); - assertThat(XdsClientImpl.matchHostName("foo-bar", pattern)).isTrue(); - } - - @Test - public void findVirtualHostForHostName_exactMatchFirst() { - String hostname = "a.googleapis.com"; - VirtualHost vHost1 = - VirtualHost.newBuilder() - .setName("virtualhost01.googleapis.com") // don't care - .addAllDomains(ImmutableList.of("a.googleapis.com", "b.googleapis.com")) - .build(); - VirtualHost vHost2 = - VirtualHost.newBuilder() - .setName("virtualhost02.googleapis.com") // don't care - .addAllDomains(ImmutableList.of("*.googleapis.com")) - .build(); - VirtualHost vHost3 = - VirtualHost.newBuilder() - .setName("virtualhost03.googleapis.com") // don't care - .addAllDomains(ImmutableList.of("*")) - .build(); - RouteConfiguration routeConfig = - RouteConfiguration.newBuilder() - .setName("route-foo.googleapis.com") - .addAllVirtualHosts(ImmutableList.of(vHost1, vHost2, vHost3)) - .build(); - assertThat(XdsClientImpl.findVirtualHostForHostName(routeConfig, hostname)).isEqualTo(vHost1); - } - - @Test - public void findVirtualHostForHostName_preferSuffixDomainOverPrefixDomain() { - String hostname = "a.googleapis.com"; - VirtualHost vHost1 = - VirtualHost.newBuilder() - .setName("virtualhost01.googleapis.com") // don't care - .addAllDomains(ImmutableList.of("*.googleapis.com", "b.googleapis.com")) - .build(); - VirtualHost vHost2 = - VirtualHost.newBuilder() - .setName("virtualhost02.googleapis.com") // don't care - .addAllDomains(ImmutableList.of("a.googleapis.*")) - .build(); - VirtualHost vHost3 = - VirtualHost.newBuilder() - .setName("virtualhost03.googleapis.com") // don't care - .addAllDomains(ImmutableList.of("*")) - .build(); - RouteConfiguration routeConfig = - RouteConfiguration.newBuilder() - .setName("route-foo.googleapis.com") - .addAllVirtualHosts(ImmutableList.of(vHost1, vHost2, vHost3)) - .build(); - assertThat(XdsClientImpl.findVirtualHostForHostName(routeConfig, hostname)).isEqualTo(vHost1); - } - - @Test - public void findVirtualHostForHostName_asteriskMatchAnyDomain() { - String hostname = "a.googleapis.com"; - VirtualHost vHost1 = - VirtualHost.newBuilder() - .setName("virtualhost01.googleapis.com") // don't care - .addAllDomains(ImmutableList.of("*")) - .build(); - VirtualHost vHost2 = - VirtualHost.newBuilder() - .setName("virtualhost02.googleapis.com") // don't care - .addAllDomains(ImmutableList.of("b.googleapis.com")) - .build(); - RouteConfiguration routeConfig = - RouteConfiguration.newBuilder() - .setName("route-foo.googleapis.com") - .addAllVirtualHosts(ImmutableList.of(vHost1, vHost2)) - .build(); - assertThat(XdsClientImpl.findVirtualHostForHostName(routeConfig, hostname)).isEqualTo(vHost1); - } - - @Test - public void populateRoutesInVirtualHost_routeWithCaseInsensitiveMatch() { - VirtualHost virtualHost = - VirtualHost.newBuilder() - .setName("virtualhost00.googleapis.com") // don't care - .addDomains(TARGET_AUTHORITY) - .addRoutes( - Route.newBuilder() - .setRoute(RouteAction.newBuilder().setCluster("cluster.googleapis.com")) - .setMatch( - RouteMatch.newBuilder() - .setPrefix("") - .setCaseSensitive(BoolValue.newBuilder().setValue(false)))) - .build(); - - thrown.expect(XdsClientImpl.InvalidProtoDataException.class); - XdsClientImpl.populateRoutesInVirtualHost(virtualHost); - } - - @Test - public void populateRoutesInVirtualHost_NoUsableRoute() { - VirtualHost virtualHost = - VirtualHost.newBuilder() - .setName("virtualhost00.googleapis.com") // don't care - .addDomains(TARGET_AUTHORITY) - .addRoutes( - // route with unsupported action - Route.newBuilder() - .setRoute(RouteAction.newBuilder().setClusterHeader("cluster header string")) - .setMatch(RouteMatch.newBuilder().setPrefix("/"))) - .addRoutes( - // route with unsupported matcher type - Route.newBuilder() - .setRoute(RouteAction.newBuilder().setCluster("cluster.googleapis.com")) - .setMatch( - RouteMatch.newBuilder() - .setPrefix("/") - .addQueryParameters(QueryParameterMatcher.getDefaultInstance()))) - .build(); - - thrown.expect(XdsClientImpl.InvalidProtoDataException.class); - XdsClientImpl.populateRoutesInVirtualHost(virtualHost); - } - - @Test - public void messagePrinter_printLdsResponse() { - MessagePrinter printer = new MessagePrinter(); - List listeners = ImmutableList.of( - Any.pack(buildListenerV2("foo.googleapis.com:8080", - Any.pack( - HttpConnectionManager.newBuilder() - .setRouteConfig( - buildRouteConfigurationV2("route-foo.googleapis.com", - ImmutableList.of( - buildVirtualHostV2( - ImmutableList.of("foo.googleapis.com", "bar.googleapis.com"), - "cluster.googleapis.com")))) - .build())))); - DiscoveryResponse response = - buildDiscoveryResponseV2("0", listeners, XdsClientImpl.ADS_TYPE_URL_LDS_V2, "0000"); - - String expectedString = "{\n" - + " \"versionInfo\": \"0\",\n" - + " \"resources\": [{\n" - + " \"@type\": \"type.googleapis.com/envoy.api.v2.Listener\",\n" - + " \"name\": \"foo.googleapis.com:8080\",\n" - + " \"address\": {\n" - + " },\n" - + " \"filterChains\": [{\n" - + " }],\n" - + " \"apiListener\": {\n" - + " \"apiListener\": {\n" - + " \"@type\": \"type.googleapis.com/envoy.config.filter.network" - + ".http_connection_manager.v2.HttpConnectionManager\",\n" - + " \"routeConfig\": {\n" - + " \"name\": \"route-foo.googleapis.com\",\n" - + " \"virtualHosts\": [{\n" - + " \"name\": \"virtualhost00.googleapis.com\",\n" - + " \"domains\": [\"foo.googleapis.com\", \"bar.googleapis.com\"],\n" - + " \"routes\": [{\n" - + " \"match\": {\n" - + " \"prefix\": \"\"\n" - + " },\n" - + " \"route\": {\n" - + " \"cluster\": \"cluster.googleapis.com\"\n" - + " }\n" - + " }]\n" - + " }]\n" - + " }\n" - + " }\n" - + " }\n" - + " }],\n" - + " \"typeUrl\": \"type.googleapis.com/envoy.api.v2.Listener\",\n" - + " \"nonce\": \"0000\"\n" - + "}"; - String res = printer.print(response); - assertThat(res).isEqualTo(expectedString); - } - - @Test - public void messagePrinter_printRdsResponse() { - MessagePrinter printer = new MessagePrinter(); - List routeConfigs = - ImmutableList.of( - Any.pack( - buildRouteConfigurationV2( - "route-foo.googleapis.com", - ImmutableList.of( - buildVirtualHostV2( - ImmutableList.of("foo.googleapis.com", "bar.googleapis.com"), - "cluster.googleapis.com"))))); - DiscoveryResponse response = - buildDiscoveryResponseV2("213", routeConfigs, XdsClientImpl.ADS_TYPE_URL_RDS_V2, "0052"); - - String expectedString = "{\n" - + " \"versionInfo\": \"213\",\n" - + " \"resources\": [{\n" - + " \"@type\": \"type.googleapis.com/envoy.api.v2.RouteConfiguration\",\n" - + " \"name\": \"route-foo.googleapis.com\",\n" - + " \"virtualHosts\": [{\n" - + " \"name\": \"virtualhost00.googleapis.com\",\n" - + " \"domains\": [\"foo.googleapis.com\", \"bar.googleapis.com\"],\n" - + " \"routes\": [{\n" - + " \"match\": {\n" - + " \"prefix\": \"\"\n" - + " },\n" - + " \"route\": {\n" - + " \"cluster\": \"cluster.googleapis.com\"\n" - + " }\n" - + " }]\n" - + " }]\n" - + " }],\n" - + " \"typeUrl\": \"type.googleapis.com/envoy.api.v2.RouteConfiguration\",\n" - + " \"nonce\": \"0052\"\n" - + "}"; - String res = printer.print(response); - assertThat(res).isEqualTo(expectedString); - } - - @Test - public void messagePrinter_printCdsResponse() { - MessagePrinter printer = new MessagePrinter(); - List clusters = ImmutableList.of( - Any.pack(buildClusterV2("cluster-bar.googleapis.com", "service-blaze:cluster-bar", true)), - Any.pack(buildClusterV2("cluster-foo.googleapis.com", null, false))); - DiscoveryResponse response = - buildDiscoveryResponseV2("14", clusters, XdsClientImpl.ADS_TYPE_URL_CDS_V2, "8"); - - String expectedString = "{\n" - + " \"versionInfo\": \"14\",\n" - + " \"resources\": [{\n" - + " \"@type\": \"type.googleapis.com/envoy.api.v2.Cluster\",\n" - + " \"name\": \"cluster-bar.googleapis.com\",\n" - + " \"type\": \"EDS\",\n" - + " \"edsClusterConfig\": {\n" - + " \"edsConfig\": {\n" - + " \"ads\": {\n" - + " }\n" - + " },\n" - + " \"serviceName\": \"service-blaze:cluster-bar\"\n" - + " },\n" - + " \"lrsServer\": {\n" - + " \"self\": {\n" - + " }\n" - + " }\n" - + " }, {\n" - + " \"@type\": \"type.googleapis.com/envoy.api.v2.Cluster\",\n" - + " \"name\": \"cluster-foo.googleapis.com\",\n" - + " \"type\": \"EDS\",\n" - + " \"edsClusterConfig\": {\n" - + " \"edsConfig\": {\n" - + " \"ads\": {\n" - + " }\n" - + " }\n" - + " }\n" - + " }],\n" - + " \"typeUrl\": \"type.googleapis.com/envoy.api.v2.Cluster\",\n" - + " \"nonce\": \"8\"\n" - + "}"; - String res = printer.print(response); - assertThat(res).isEqualTo(expectedString); - } - - @Test - public void messagePrinter_printEdsResponse() { - MessagePrinter printer = new MessagePrinter(); - List clusterLoadAssignments = ImmutableList.of( - Any.pack(buildClusterLoadAssignmentV2("cluster-foo.googleapis.com", - ImmutableList.of( - buildLocalityLbEndpointsV2("region1", "zone1", "subzone1", - ImmutableList.of( - buildLbEndpointV2("192.168.0.1", 8080, HealthStatus.HEALTHY, 2)), - 1, 0), - buildLocalityLbEndpointsV2("region3", "zone3", "subzone3", - ImmutableList.of( - buildLbEndpointV2("192.168.142.5", 80, HealthStatus.UNHEALTHY, 5)), - 2, 1)), - ImmutableList.of( - buildDropOverloadV2("lb", 200), - buildDropOverloadV2("throttle", 1000))))); - - DiscoveryResponse response = - buildDiscoveryResponseV2("5", clusterLoadAssignments, - XdsClientImpl.ADS_TYPE_URL_EDS_V2, "004"); - - String expectedString = "{\n" - + " \"versionInfo\": \"5\",\n" - + " \"resources\": [{\n" - + " \"@type\": \"type.googleapis.com/envoy.api.v2.ClusterLoadAssignment\",\n" - + " \"clusterName\": \"cluster-foo.googleapis.com\",\n" - + " \"endpoints\": [{\n" - + " \"locality\": {\n" - + " \"region\": \"region1\",\n" - + " \"zone\": \"zone1\",\n" - + " \"subZone\": \"subzone1\"\n" - + " },\n" - + " \"lbEndpoints\": [{\n" - + " \"endpoint\": {\n" - + " \"address\": {\n" - + " \"socketAddress\": {\n" - + " \"address\": \"192.168.0.1\",\n" - + " \"portValue\": 8080\n" - + " }\n" - + " }\n" - + " },\n" - + " \"healthStatus\": \"HEALTHY\",\n" - + " \"loadBalancingWeight\": 2\n" - + " }],\n" - + " \"loadBalancingWeight\": 1\n" - + " }, {\n" - + " \"locality\": {\n" - + " \"region\": \"region3\",\n" - + " \"zone\": \"zone3\",\n" - + " \"subZone\": \"subzone3\"\n" - + " },\n" - + " \"lbEndpoints\": [{\n" - + " \"endpoint\": {\n" - + " \"address\": {\n" - + " \"socketAddress\": {\n" - + " \"address\": \"192.168.142.5\",\n" - + " \"portValue\": 80\n" - + " }\n" - + " }\n" - + " },\n" - + " \"healthStatus\": \"UNHEALTHY\",\n" - + " \"loadBalancingWeight\": 5\n" - + " }],\n" - + " \"loadBalancingWeight\": 2,\n" - + " \"priority\": 1\n" - + " }],\n" - + " \"policy\": {\n" - + " \"dropOverloads\": [{\n" - + " \"category\": \"lb\",\n" - + " \"dropPercentage\": {\n" - + " \"numerator\": 200,\n" - + " \"denominator\": \"MILLION\"\n" - + " }\n" - + " }, {\n" - + " \"category\": \"throttle\",\n" - + " \"dropPercentage\": {\n" - + " \"numerator\": 1000,\n" - + " \"denominator\": \"MILLION\"\n" - + " }\n" - + " }],\n" - + " \"disableOverprovisioning\": true\n" - + " }\n" - + " }],\n" - + " \"typeUrl\": \"type.googleapis.com/envoy.api.v2.ClusterLoadAssignment\",\n" - + " \"nonce\": \"004\"\n" - + "}"; - String res = printer.print(response); - assertThat(res).isEqualTo(expectedString); - } - - private static void assertConfigUpdateContainsSingleClusterRoute( - ConfigUpdate configUpdate, String expectedClusterName) { - List routes = configUpdate.getRoutes(); - assertThat(routes).hasSize(1); - assertThat(Iterables.getOnlyElement(routes).getRouteAction().getCluster()) - .isEqualTo(expectedClusterName); - } - - /** - * Matcher for DiscoveryRequest without the comparison of error_details field, which is used for - * management server debugging purposes. - * - *

In general, if you are sure error_details field should not be set in a DiscoveryRequest, - * compare with message equality. Otherwise, this matcher is handy for comparing other fields - * only. - */ - private static class DiscoveryRequestMatcher implements ArgumentMatcher { - private final String versionInfo; - private final String typeUrl; - private final Set resourceNames; - private final String responseNonce; - - private DiscoveryRequestMatcher(String versionInfo, String resourceName, String typeUrl, - String responseNonce) { - this(versionInfo, ImmutableList.of(resourceName), typeUrl, responseNonce); - } - - private DiscoveryRequestMatcher( - String versionInfo, List resourceNames, String typeUrl, String responseNonce) { - this.versionInfo = versionInfo; - this.resourceNames = new HashSet<>(resourceNames); - this.typeUrl = typeUrl; - this.responseNonce = responseNonce; - } - - @Override - public boolean matches(DiscoveryRequest argument) { - if (!typeUrl.equals(argument.getTypeUrl())) { - return false; - } - if (!versionInfo.equals(argument.getVersionInfo())) { - return false; - } - if (!responseNonce.equals(argument.getResponseNonce())) { - return false; - } - if (!resourceNames.equals(new HashSet<>(argument.getResourceNamesList()))) { - return false; - } - return argument.getNode().equals(NODE.toEnvoyProtoNodeV2()); - } - } - - private static class LoadReportCall { - private final StreamObserver requestObserver; - @SuppressWarnings("unused") - private final StreamObserver responseObserver; - - LoadReportCall(StreamObserver requestObserver, - StreamObserver responseObserver) { - this.requestObserver = requestObserver; - this.responseObserver = responseObserver; - } - } -} From 2adeff56fe64e36e75f5f299a957666266c02ff3 Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Mon, 28 Sep 2020 13:43:41 -0700 Subject: [PATCH 72/86] xds: refactor resource subscription implementation in XdsClient (#7458) Introduce ResourceSubscriber for tracking the state of a single resource. Every time newly subscribing to some resource, a corresponding ResourceSubscriber is created. Note it does not control the resource discovery RPCs. It is still the XdsClient that sends RPCs for with all subscribed resource names for each type. A ResourceSubscriber can have the following states: - When the initial resource fetch timer (respTimer) is pending, the resource is under discovery, the resource data is unknown. Even if the XdsClient receives a response not containing the corresponding resource, it does not mean the resource is absent. We still need to wait until a response containing the resource data coming or the timer being fired. The timer is scheduled when the ResourceSubscriber is created. So the XdsClient should always create the corresponding ResourceSubscriber when it starts to subscribe a new resource. - If the resource fetch timer is not pending, we must know the existence of the resource data. If data field is set, it is the most recently received resource data (aka, cached entry). Otherwise, absent field is set to true, indicating the resource does not exist. The exceptional case is when the ADS stream is closed and in the retry backoff period. During that period, respTimer is cancelled and the resource existence may or may not be known. Once the backoff finishes, the XdsClient will reschedule the respTimer when it recreates the ADS stream and re-request all the resources. Watchers can be added to existing ResourceSubscribers. At the time the watcher is added, its callback will be invoked if we've already known the existence of the resource. Otherwise, the watcher will just sit there and wait data or absence to come in the future. --- xds/src/main/java/io/grpc/xds/XdsClient.java | 50 +- .../main/java/io/grpc/xds/XdsClientImpl.java | 529 ++++++++---------- .../java/io/grpc/xds/XdsClientImplTest.java | 8 +- 3 files changed, 257 insertions(+), 330 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/XdsClient.java b/xds/src/main/java/io/grpc/xds/XdsClient.java index 7ec7afdc8b4..01a879a2481 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClient.java +++ b/xds/src/main/java/io/grpc/xds/XdsClient.java @@ -103,7 +103,7 @@ ConfigUpdate build() { } } - static final class LdsUpdate { + static final class LdsUpdate implements ResourceUpdate { // Total number of nanoseconds to keep alive an HTTP request/response stream. private final long httpMaxStreamDurationNano; // The name of the route configuration to be used for RDS resource discovery. @@ -134,6 +134,25 @@ List getVirtualHosts() { return virtualHosts; } + @Override + public int hashCode() { + return Objects.hash(httpMaxStreamDurationNano, rdsName, virtualHosts); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LdsUpdate that = (LdsUpdate) o; + return Objects.equals(httpMaxStreamDurationNano, that.httpMaxStreamDurationNano) + && Objects.equals(rdsName, that.rdsName) + && Objects.equals(virtualHosts, that.virtualHosts); + } + @Override public String toString() { ToStringHelper toStringHelper = MoreObjects.toStringHelper(this); @@ -182,7 +201,7 @@ LdsUpdate build() { } } - static final class RdsUpdate { + static final class RdsUpdate implements ResourceUpdate { // The list virtual hosts that make up the route table. private final List virtualHosts; @@ -206,7 +225,7 @@ public String toString() { } } - static final class CdsUpdate { + static final class CdsUpdate implements ResourceUpdate { private final String clusterName; @Nullable private final String edsServiceName; @@ -351,7 +370,7 @@ CdsUpdate build() { } } - static final class EdsUpdate { + static final class EdsUpdate implements ResourceUpdate { private final String clusterName; private final Map localityLbEndpointsMap; private final List dropPolicies; @@ -497,10 +516,13 @@ ListenerUpdate build() { } } + interface ResourceUpdate { + } + /** * Watcher interface for a single requested xDS resource. */ - private interface ResourceWatcher { + interface ResourceWatcher { /** * Called when the resource discovery RPC encounters some transient error. @@ -523,6 +545,14 @@ interface RdsResourceWatcher extends ResourceWatcher { void onChanged(RdsUpdate update); } + interface CdsResourceWatcher extends ResourceWatcher { + void onChanged(CdsUpdate update); + } + + interface EdsResourceWatcher extends ResourceWatcher { + void onChanged(EdsUpdate update); + } + /** * Config watcher interface. To be implemented by the xDS resolver. */ @@ -535,16 +565,6 @@ interface ConfigWatcher extends ResourceWatcher { void onConfigChanged(ConfigUpdate update); } - interface CdsResourceWatcher extends ResourceWatcher { - - void onChanged(CdsUpdate update); - } - - interface EdsResourceWatcher extends ResourceWatcher { - - void onChanged(EdsUpdate update); - } - /** * Listener watcher interface. To be used by {@link io.grpc.xds.internal.sds.XdsServerBuilder}. */ diff --git a/xds/src/main/java/io/grpc/xds/XdsClientImpl.java b/xds/src/main/java/io/grpc/xds/XdsClientImpl.java index b01b01acc79..507aeca322a 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientImpl.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientImpl.java @@ -69,6 +69,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -126,39 +127,8 @@ final class XdsClientImpl extends XdsClient { // more than once. private Node node; - // Cached data for CDS responses, keyed by cluster names. - // Optimization: cache CdsUpdate, which contains only information needed by gRPC, instead - // of whole Cluster messages to reduce memory usage. - private final Map clusterNamesToCdsUpdates = new HashMap<>(); - - // Cached CDS resources that are known to be absent. - private final Set absentCdsResources = new HashSet<>(); - - // Cached data for EDS responses, keyed by cluster names. - // CDS responses indicate absence of clusters and EDS responses indicate presence of clusters. - // Optimization: cache EdsUpdate, which contains only information needed by gRPC, instead - // of whole ClusterLoadAssignment messages to reduce memory usage. - private final Map clusterNamesToEdsUpdates = new HashMap<>(); - - // Cached EDS resources that are known to be absent. - private final Set absentEdsResources = new HashSet<>(); - - // Cluster watchers waiting for cluster information updates. Multiple cluster watchers - // can watch on information for the same cluster. - private final Map> cdsWatchers = new HashMap<>(); - - // Endpoint watchers waiting for endpoint updates for each cluster. Multiple endpoint - // watchers can watch endpoints in the same cluster. - private final Map> edsWatchers = new HashMap<>(); - - // Resource fetch timers are used to conclude absence of resources. Each timer is activated when - // subscription for the resource starts and disarmed on first update for the resource. - - // Timers for concluding CDS resources not found. - private final Map cdsRespTimers = new HashMap<>(); - - // Timers for concluding EDS resources not found. - private final Map edsRespTimers = new HashMap<>(); + private final Map cdsResourceSubscribers = new HashMap<>(); + private final Map edsResourceSubscribers = new HashMap<>(); private final LoadStatsManager loadStatsManager = new LoadStatsManager(); @@ -248,14 +218,12 @@ private void cleanUpResourceTimers() { rdsRespTimer.cancel(); rdsRespTimer = null; } - for (ScheduledHandle handle : cdsRespTimers.values()) { - handle.cancel(); + for (ResourceSubscriber subscriber : cdsResourceSubscribers.values()) { + subscriber.stopTimer(); } - cdsRespTimers.clear(); - for (ScheduledHandle handle : edsRespTimers.values()) { - handle.cancel(); + for (ResourceSubscriber subscriber : edsResourceSubscribers.values()) { + subscriber.stopTimer(); } - edsRespTimers.clear(); } @Override @@ -282,151 +250,49 @@ void watchConfigData(String targetAuthority, ConfigWatcher watcher) { @Override void watchCdsResource(String resourceName, CdsResourceWatcher watcher) { - checkNotNull(resourceName, "resourceName"); - checkNotNull(watcher, "watcher"); - boolean needRequest = false; - if (!cdsWatchers.containsKey(resourceName)) { - logger.log(XdsLogLevel.INFO, "Start watching cluster {0}", resourceName); - needRequest = true; - cdsWatchers.put(resourceName, new HashSet()); - } - Set watchers = cdsWatchers.get(resourceName); - checkState(!watchers.contains(watcher), "watcher for %s already registered", resourceName); - watchers.add(watcher); - // If local cache contains cluster information to be watched, notify the watcher immediately. - if (absentCdsResources.contains(resourceName)) { - logger.log(XdsLogLevel.DEBUG, "Cluster resource {0} is known to be absent", resourceName); - watcher.onResourceDoesNotExist(resourceName); - return; - } - if (clusterNamesToCdsUpdates.containsKey(resourceName)) { - logger.log(XdsLogLevel.DEBUG, "Retrieve cluster info {0} from local cache", resourceName); - watcher.onChanged(clusterNamesToCdsUpdates.get(resourceName)); - return; - } - - if (needRequest) { - if (rpcRetryTimer != null && rpcRetryTimer.isPending()) { - // Currently in retry backoff. - return; - } - if (adsStream == null) { - startRpcStream(); - } - adsStream.sendXdsRequest(ResourceType.CDS, cdsWatchers.keySet()); - ScheduledHandle timeoutHandle = - syncContext - .schedule( - new CdsResourceFetchTimeoutTask(resourceName), - INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS, timeService); - cdsRespTimers.put(resourceName, timeoutHandle); - } + ResourceSubscriber subscriber = cdsResourceSubscribers.get(resourceName); + if (subscriber == null) { + logger.log(XdsLogLevel.INFO, "Subscribe CDS resource {0}", resourceName); + subscriber = new ResourceSubscriber(ResourceType.CDS, resourceName); + cdsResourceSubscribers.put(resourceName, subscriber); + adjustResourceSubscription(ResourceType.CDS, cdsResourceSubscribers.keySet()); + } + subscriber.addWatcher(watcher); } @Override void cancelCdsResourceWatch(String resourceName, CdsResourceWatcher watcher) { - checkNotNull(watcher, "watcher"); - Set watchers = cdsWatchers.get(resourceName); - checkState( - watchers != null && watchers.contains(watcher), - "watcher for %s was not registered", resourceName); - watchers.remove(watcher); - if (watchers.isEmpty()) { - logger.log(XdsLogLevel.INFO, "Stop watching cluster {0}", resourceName); - cdsWatchers.remove(resourceName); - // Remove the corresponding CDS entry. - absentCdsResources.remove(resourceName); - clusterNamesToCdsUpdates.remove(resourceName); - // Cancel and delete response timer waiting for the corresponding resource. - if (cdsRespTimers.containsKey(resourceName)) { - cdsRespTimers.get(resourceName).cancel(); - cdsRespTimers.remove(resourceName); - } - // No longer interested in this cluster, send an updated CDS request to unsubscribe - // this resource. - if (rpcRetryTimer != null && rpcRetryTimer.isPending()) { - // Currently in retry backoff. - return; - } - checkState(adsStream != null, - "Severe bug: ADS stream was not created while an endpoint watcher was registered"); - adsStream.sendXdsRequest(ResourceType.CDS, cdsWatchers.keySet()); + ResourceSubscriber subscriber = cdsResourceSubscribers.get(resourceName); + subscriber.removeWatcher(watcher); + if (!subscriber.isWatched()) { + subscriber.stopTimer(); + logger.log(XdsLogLevel.INFO, "Unsubscribe CDS resource {0}", resourceName); + cdsResourceSubscribers.remove(resourceName); + adjustResourceSubscription(ResourceType.CDS, cdsResourceSubscribers.keySet()); } } @Override void watchEdsResource(String resourceName, EdsResourceWatcher watcher) { - checkNotNull(watcher, "watcher"); - boolean needRequest = false; - if (!edsWatchers.containsKey(resourceName)) { - logger.log(XdsLogLevel.INFO, "Start watching endpoints in cluster {0}", resourceName); - needRequest = true; - edsWatchers.put(resourceName, new HashSet()); - } - Set watchers = edsWatchers.get(resourceName); - checkState(!watchers.contains(watcher), "watcher for %s already registered", resourceName); - watchers.add(watcher); - // If local cache contains endpoint information for the cluster to be watched, notify - // the watcher immediately. - if (absentEdsResources.contains(resourceName)) { - logger.log( - XdsLogLevel.DEBUG, - "Endpoint resource for cluster {0} is known to be absent.", resourceName); - watcher.onResourceDoesNotExist(resourceName); - return; - } - if (clusterNamesToEdsUpdates.containsKey(resourceName)) { - logger.log( - XdsLogLevel.DEBUG, - "Retrieve endpoints info for cluster {0} from local cache.", resourceName); - watcher.onChanged(clusterNamesToEdsUpdates.get(resourceName)); - return; - } - - if (needRequest) { - if (rpcRetryTimer != null && rpcRetryTimer.isPending()) { - // Currently in retry backoff. - return; - } - if (adsStream == null) { - startRpcStream(); - } - adsStream.sendXdsRequest(ResourceType.EDS, edsWatchers.keySet()); - ScheduledHandle timeoutHandle = - syncContext - .schedule( - new EdsResourceFetchTimeoutTask(resourceName), - INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS, timeService); - edsRespTimers.put(resourceName, timeoutHandle); - } + ResourceSubscriber subscriber = edsResourceSubscribers.get(resourceName); + if (subscriber == null) { + logger.log(XdsLogLevel.INFO, "Subscribe EDS resource {0}", resourceName); + subscriber = new ResourceSubscriber(ResourceType.EDS, resourceName); + edsResourceSubscribers.put(resourceName, subscriber); + adjustResourceSubscription(ResourceType.EDS, edsResourceSubscribers.keySet()); + } + subscriber.addWatcher(watcher); } @Override void cancelEdsResourceWatch(String resourceName, EdsResourceWatcher watcher) { - checkNotNull(watcher, "watcher"); - Set watchers = edsWatchers.get(resourceName); - checkState( - watchers != null && watchers.contains(watcher), - "watcher for %s was not registered", resourceName); - watchers.remove(watcher); - if (watchers.isEmpty()) { - logger.log(XdsLogLevel.INFO, "Stop watching endpoints in cluster {0}", resourceName); - edsWatchers.remove(resourceName); - // Remove the corresponding EDS cache entry. - absentEdsResources.remove(resourceName); - clusterNamesToEdsUpdates.remove(resourceName); - // Cancel and delete response timer waiting for the corresponding resource. - if (edsRespTimers.containsKey(resourceName)) { - edsRespTimers.get(resourceName).cancel(); - edsRespTimers.remove(resourceName); - } - // No longer interested in this cluster, send an updated EDS request to unsubscribe - // this resource. - if (rpcRetryTimer != null && rpcRetryTimer.isPending()) { - // Currently in retry backoff. - return; - } - adsStream.sendXdsRequest(ResourceType.EDS, edsWatchers.keySet()); + ResourceSubscriber subscriber = edsResourceSubscribers.get(resourceName); + subscriber.removeWatcher(watcher); + if (!subscriber.isWatched()) { + subscriber.stopTimer(); + logger.log(XdsLogLevel.INFO, "Unsubscribe EDS resource {0}", resourceName); + edsResourceSubscribers.remove(resourceName); + adjustResourceSubscription(ResourceType.EDS, edsResourceSubscribers.keySet()); } } @@ -949,7 +815,7 @@ private void handleCdsResponse(DiscoveryResponseData cdsResponse) { } catch (InvalidProtocolBufferException e) { logger.log(XdsLogLevel.WARNING, "Failed to unpack Clusters in CDS response {0}", e); adsStream.sendNackRequest( - ResourceType.CDS, cdsWatchers.keySet(), + ResourceType.CDS, cdsResourceSubscribers.keySet(), cdsResponse.getVersionInfo(), "Malformed CDS response: " + e); return; } @@ -967,7 +833,7 @@ private void handleCdsResponse(DiscoveryResponseData cdsResponse) { // Management server is required to always send newly requested resources, even if they // may have been sent previously (proactively). Thus, client does not need to cache // unrequested resources. - if (!cdsWatchers.containsKey(clusterName)) { + if (!cdsResourceSubscribers.containsKey(clusterName)) { continue; } CdsUpdate.Builder updateBuilder = CdsUpdate.newBuilder(); @@ -1025,63 +891,26 @@ private void handleCdsResponse(DiscoveryResponseData cdsResponse) { if (errorMessage != null) { adsStream.sendNackRequest( ResourceType.CDS, - cdsWatchers.keySet(), + cdsResourceSubscribers.keySet(), cdsResponse.getVersionInfo(), errorMessage); return; } - adsStream.sendAckRequest(ResourceType.CDS, cdsWatchers.keySet(), + adsStream.sendAckRequest(ResourceType.CDS, cdsResourceSubscribers.keySet(), cdsResponse.getVersionInfo()); - // Update local CDS cache with data in this response. - absentCdsResources.removeAll(cdsUpdates.keySet()); - for (Map.Entry entry : clusterNamesToCdsUpdates.entrySet()) { - if (!cdsUpdates.containsKey(entry.getKey())) { - // Some previously existing resource no longer exists. - absentCdsResources.add(entry.getKey()); - } else if (cdsUpdates.get(entry.getKey()).equals(entry.getValue())) { - cdsUpdates.remove(entry.getKey()); - } - } - clusterNamesToCdsUpdates.keySet().removeAll(absentCdsResources); - clusterNamesToCdsUpdates.putAll(cdsUpdates); - - // Remove EDS cache entries for ClusterLoadAssignments not referenced by this CDS response. - for (String clusterName : clusterNamesToEdsUpdates.keySet()) { - if (!edsServices.contains(clusterName)) { - absentEdsResources.add(clusterName); - // Notify EDS resource removal to watchers. - if (edsWatchers.containsKey(clusterName)) { - Set watchers = edsWatchers.get(clusterName); - for (EdsResourceWatcher watcher : watchers) { - watcher.onResourceDoesNotExist(clusterName); - } - } - } - } - clusterNamesToEdsUpdates.keySet().retainAll(edsServices); - - for (String clusterName : cdsUpdates.keySet()) { - if (cdsRespTimers.containsKey(clusterName)) { - cdsRespTimers.get(clusterName).cancel(); - cdsRespTimers.remove(clusterName); + for (String resource : cdsResourceSubscribers.keySet()) { + ResourceSubscriber subscriber = cdsResourceSubscribers.get(resource); + if (cdsUpdates.containsKey(resource)) { + subscriber.onData(cdsUpdates.get(resource)); + } else { + subscriber.onAbsent(); } } - - // Notify watchers if clusters interested in present in this CDS response. - for (Map.Entry> entry : cdsWatchers.entrySet()) { - String clusterName = entry.getKey(); - if (cdsUpdates.containsKey(entry.getKey())) { - CdsUpdate cdsUpdate = cdsUpdates.get(clusterName); - for (CdsResourceWatcher watcher : entry.getValue()) { - watcher.onChanged(cdsUpdate); - } - } else if (!clusterNamesToCdsUpdates.containsKey(entry.getKey()) - && !cdsRespTimers.containsKey(clusterName)) { - // Update for previously present resource being removed. - for (CdsResourceWatcher watcher : entry.getValue()) { - watcher.onResourceDoesNotExist(entry.getKey()); - } + for (String resource : edsResourceSubscribers.keySet()) { + if (!edsServices.contains(resource)) { + ResourceSubscriber subscriber = edsResourceSubscribers.get(resource); + subscriber.onAbsent(); } } } @@ -1124,7 +953,7 @@ private void handleEdsResponse(DiscoveryResponseData edsResponse) { logger.log( XdsLogLevel.WARNING, "Failed to unpack ClusterLoadAssignments in EDS response {0}", e); adsStream.sendNackRequest( - ResourceType.EDS, edsWatchers.keySet(), + ResourceType.EDS, edsResourceSubscribers.keySet(), edsResponse.getVersionInfo(), "Malformed EDS response: " + e); return; } @@ -1141,7 +970,7 @@ private void handleEdsResponse(DiscoveryResponseData edsResponse) { // Management server is required to always send newly requested resources, even if they // may have been sent previously (proactively). Thus, client does not need to cache // unrequested resources. - if (!edsWatchers.containsKey(clusterName)) { + if (!edsResourceSubscribers.containsKey(clusterName)) { continue; } EdsUpdate.Builder updateBuilder = EdsUpdate.newBuilder(); @@ -1198,36 +1027,33 @@ private void handleEdsResponse(DiscoveryResponseData edsResponse) { if (errorMessage != null) { adsStream.sendNackRequest( ResourceType.EDS, - edsWatchers.keySet(), + edsResourceSubscribers.keySet(), edsResponse.getVersionInfo(), errorMessage); return; } - adsStream.sendAckRequest(ResourceType.EDS, edsWatchers.keySet(), + adsStream.sendAckRequest(ResourceType.EDS, edsResourceSubscribers.keySet(), edsResponse.getVersionInfo()); - // Update local EDS cache by inserting updated endpoint information. - clusterNamesToEdsUpdates.putAll(edsUpdates); - absentEdsResources.removeAll(edsUpdates.keySet()); - - // Notify watchers waiting for updates of endpoint information received in this EDS response. - // Based on xDS protocol, the management server should not send endpoint data again if - // nothing has changed. - for (Map.Entry entry : edsUpdates.entrySet()) { - String clusterName = entry.getKey(); - // Cancel and delete response timeout timer. - if (edsRespTimers.containsKey(clusterName)) { - edsRespTimers.get(clusterName).cancel(); - edsRespTimers.remove(clusterName); - } - if (edsWatchers.containsKey(clusterName)) { - for (EdsResourceWatcher watcher : edsWatchers.get(clusterName)) { - watcher.onChanged(entry.getValue()); - } + for (String resource : edsResourceSubscribers.keySet()) { + ResourceSubscriber subscriber = edsResourceSubscribers.get(resource); + if (edsUpdates.containsKey(resource)) { + subscriber.onData(edsUpdates.get(resource)); } } } + private void adjustResourceSubscription(ResourceType type, Collection resources) { + if (rpcRetryTimer != null && rpcRetryTimer.isPending()) { + // Currently in retry backoff. + return; + } + if (adsStream == null) { + startRpcStream(); + } + adsStream.sendXdsRequest(type, resources); + } + @VisibleForTesting final class RpcRetryTask implements Runnable { @Override @@ -1249,35 +1075,26 @@ public void run() { new ListenerResourceFetchTimeoutTask(":" + listenerPort), INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS, timeService); } - if (!cdsWatchers.isEmpty()) { - adsStream.sendXdsRequest(ResourceType.CDS, cdsWatchers.keySet()); - for (String clusterName : cdsWatchers.keySet()) { - ScheduledHandle timeoutHandle = - syncContext - .schedule( - new CdsResourceFetchTimeoutTask(clusterName), - INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS, timeService); - cdsRespTimers.put(clusterName, timeoutHandle); + if (!cdsResourceSubscribers.isEmpty()) { + adsStream.sendXdsRequest(ResourceType.CDS, cdsResourceSubscribers.keySet()); + for (ResourceSubscriber subscriber : cdsResourceSubscribers.values()) { + subscriber.restartTimer(); } } - if (!edsWatchers.isEmpty()) { - adsStream.sendXdsRequest(ResourceType.EDS, edsWatchers.keySet()); - for (String clusterName : edsWatchers.keySet()) { - ScheduledHandle timeoutHandle = - syncContext - .schedule( - new EdsResourceFetchTimeoutTask(clusterName), - INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS, timeService); - edsRespTimers.put(clusterName, timeoutHandle); + if (!edsResourceSubscribers.isEmpty()) { + adsStream.sendXdsRequest(ResourceType.EDS, edsResourceSubscribers.keySet()); + for (ResourceSubscriber subscriber : edsResourceSubscribers.values()) { + subscriber.restartTimer(); } } } } - private enum ResourceType { + @VisibleForTesting + enum ResourceType { UNKNOWN, LDS, RDS, CDS, EDS; - String typeUrl() { + private String typeUrl() { switch (this) { case LDS: return ADS_TYPE_URL_LDS; @@ -1293,7 +1110,7 @@ String typeUrl() { } } - String typeUrlV2() { + private String typeUrlV2() { switch (this) { case LDS: return ADS_TYPE_URL_LDS_V2; @@ -1309,7 +1126,7 @@ String typeUrlV2() { } } - static ResourceType fromTypeUrl(String typeUrl) { + private static ResourceType fromTypeUrl(String typeUrl) { switch (typeUrl) { case ADS_TYPE_URL_LDS: // fall trough @@ -1333,6 +1150,135 @@ static ResourceType fromTypeUrl(String typeUrl) { } } + /** + * Tracks a single subscribed resource. + */ + private final class ResourceSubscriber { + private final ResourceType type; + private final String resource; + private final Set watchers = new HashSet<>(); + // Resource states: + // - present: data != null; data is the cached data for the resource + // - absent: absent == true + // - unknown: anything else + // Note absent -> data == null, but not vice versa. + private ResourceUpdate data; + private boolean absent; + private ScheduledHandle respTimer; + + ResourceSubscriber(ResourceType type, String resource) { + this.type = type; + this.resource = resource; + if (rpcRetryTimer != null && rpcRetryTimer.isPending()) { + return; + } + restartTimer(); + } + + void addWatcher(ResourceWatcher watcher) { + checkArgument(!watchers.contains(watcher), "watcher %s already registered", watcher); + watchers.add(watcher); + if (data != null) { + notifyWatcher(watcher, data); + } else if (absent) { + watcher.onResourceDoesNotExist(resource); + } + } + + void removeWatcher(ResourceWatcher watcher) { + checkArgument(watchers.contains(watcher), "watcher %s not registered", watcher); + watchers.remove(watcher); + } + + void restartTimer() { + class ResourceNotFound implements Runnable { + @Override + public void run() { + respTimer = null; + onAbsent(); + } + + @Override + public String toString() { + return type + this.getClass().getSimpleName(); + } + } + + respTimer = syncContext.schedule( + new ResourceNotFound(), INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS, + timeService); + } + + void stopTimer() { + if (respTimer != null && respTimer.isPending()) { + respTimer.cancel(); + respTimer = null; + } + } + + boolean isWatched() { + return !watchers.isEmpty(); + } + + void onData(ResourceUpdate data) { + if (respTimer != null && respTimer.isPending()) { + respTimer.cancel(); + respTimer = null; + } + ResourceUpdate oldData = this.data; + this.data = data; + absent = false; + if (!Objects.equals(oldData, data)) { + for (ResourceWatcher watcher : watchers) { + notifyWatcher(watcher, data); + } + } + } + + void onAbsent() { + if (respTimer != null && respTimer.isPending()) { // too early to conclude absence + return; + } + if (!absent) { + data = null; + absent = true; + for (ResourceWatcher watcher : watchers) { + watcher.onResourceDoesNotExist(resource); + } + } + } + + void onError(Status error) { + if (respTimer != null && respTimer.isPending()) { + respTimer.cancel(); + respTimer = null; + } + for (ResourceWatcher watcher : watchers) { + watcher.onError(error); + } + } + + private void notifyWatcher(ResourceWatcher watcher, ResourceUpdate update) { + switch (type) { + case LDS: + ((LdsResourceWatcher) watcher).onChanged((LdsUpdate) update); + break; + case RDS: + ((RdsResourceWatcher) watcher).onChanged((RdsUpdate) update); + break; + case CDS: + ((CdsResourceWatcher) watcher).onChanged((CdsUpdate) update); + break; + case EDS: + ((EdsResourceWatcher) watcher).onChanged((EdsUpdate) update); + break; + case UNKNOWN: + default: + throw new AssertionError("should never be here"); + } + } + } + private static final class DiscoveryRequestData { private final ResourceType resourceType; private final Collection resourceNames; @@ -1521,15 +1467,11 @@ private void handleStreamClosed(Status error) { if (listenerWatcher != null) { listenerWatcher.onError(error); } - for (Set watchers : cdsWatchers.values()) { - for (CdsResourceWatcher watcher : watchers) { - watcher.onError(error); - } + for (ResourceSubscriber subscriber : cdsResourceSubscribers.values()) { + subscriber.onError(error); } - for (Set watchers : edsWatchers.values()) { - for (EdsResourceWatcher watcher : watchers) { - watcher.onError(error); - } + for (ResourceSubscriber subscriber : edsResourceSubscribers.values()) { + subscriber.onError(error); } cleanUp(); cleanUpResourceTimers(); @@ -1837,6 +1779,7 @@ void sendError(Exception error) { } } + // TODO(chengyuanzhang): delete me. private abstract class ResourceFetchTimeoutTask implements Runnable { final String resourceName; @@ -1853,6 +1796,7 @@ public void run() { } } + // TODO(chengyuanzhang): delete me. @VisibleForTesting final class LdsResourceFetchTimeoutTask extends ResourceFetchTimeoutTask { @@ -1883,6 +1827,7 @@ public void run() { } } + // TODO(chengyuanzhang): delete me. @VisibleForTesting final class RdsResourceFetchTimeoutTask extends ResourceFetchTimeoutTask { @@ -1898,42 +1843,6 @@ public void run() { } } - @VisibleForTesting - final class CdsResourceFetchTimeoutTask extends ResourceFetchTimeoutTask { - - CdsResourceFetchTimeoutTask(String resourceName) { - super(resourceName); - } - - @Override - public void run() { - super.run(); - cdsRespTimers.remove(resourceName); - absentCdsResources.add(resourceName); - for (CdsResourceWatcher wat : cdsWatchers.get(resourceName)) { - wat.onResourceDoesNotExist(resourceName); - } - } - } - - @VisibleForTesting - final class EdsResourceFetchTimeoutTask extends ResourceFetchTimeoutTask { - - EdsResourceFetchTimeoutTask(String resourceName) { - super(resourceName); - } - - @Override - public void run() { - super.run(); - edsRespTimers.remove(resourceName); - absentEdsResources.add(resourceName); - for (EdsResourceWatcher wat : edsWatchers.get(resourceName)) { - wat.onResourceDoesNotExist(resourceName); - } - } - } - /** * Returns {@code true} iff {@code hostName} matches the domain name {@code pattern} with * case-insensitive. diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java index 346c7230621..c0f48873edf 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java @@ -99,6 +99,7 @@ import io.grpc.xds.XdsClient.EdsUpdate; import io.grpc.xds.XdsClient.XdsChannel; import io.grpc.xds.XdsClientImpl.MessagePrinter; +import io.grpc.xds.XdsClientImpl.ResourceType; import java.io.IOException; import java.util.ArrayDeque; import java.util.HashSet; @@ -161,8 +162,7 @@ public boolean shouldAccept(Runnable command) { new TaskFilter() { @Override public boolean shouldAccept(Runnable command) { - return command.toString() - .contains(XdsClientImpl.CdsResourceFetchTimeoutTask.class.getSimpleName()); + return command.toString().contains(ResourceType.CDS.toString()); } }; @@ -170,8 +170,7 @@ public boolean shouldAccept(Runnable command) { new FakeClock.TaskFilter() { @Override public boolean shouldAccept(Runnable command) { - return command.toString() - .contains(XdsClientImpl.EdsResourceFetchTimeoutTask.class.getSimpleName()); + return command.toString().contains(ResourceType.EDS.toString()); } }; @@ -1794,7 +1793,6 @@ public void addRemoveCdsWatcherWhileInitialResourceFetchInProgress() { // Streaming RPC starts after a first watcher is added. StreamObserver requestObserver = requestObservers.poll(); - // Client sends an EDS request to management server. verify(requestObserver) .onNext( argThat( From 4c1bab9ed5ab2f01bd68081c337404726eec39e7 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Mon, 28 Sep 2020 13:19:56 -0700 Subject: [PATCH 73/86] Prepare for JUnit 4.13 It deprecates ExpectedException and Assert.assertThat(T, org.hamcrest.Matcher). Without Java 8 we don't want to migrate away from ExpectedException at this time. We tend to prefer Truth over Hamcrest, so I swapped the one instance of Assert.assertThat() to use Truth. With this change we get a warning-less build with JUnit 4.13. We don't yet upgrade because we still need to support JUnit 4.12 for some use-cases, but will be able to upgrade to 4.13 soon when they upgrade. --- api/src/test/java/io/grpc/ContextsTest.java | 5 ++--- api/src/test/java/io/grpc/MetadataTest.java | 1 + api/src/test/java/io/grpc/MethodDescriptorTest.java | 1 + api/src/test/java/io/grpc/ServerInterceptorsTest.java | 1 + api/src/test/java/io/grpc/ServerServiceDefinitionTest.java | 1 + api/src/test/java/io/grpc/ServiceDescriptorTest.java | 1 + .../test/java/io/grpc/internal/AbstractClientStreamTest.java | 1 + .../test/java/io/grpc/internal/AbstractServerStreamTest.java | 1 + .../test/java/io/grpc/internal/AbstractTransportTest.java | 1 + .../java/io/grpc/internal/ConnectivityStateManagerTest.java | 1 + core/src/test/java/io/grpc/internal/DnsNameResolverTest.java | 1 + core/src/test/java/io/grpc/internal/GrpcUtilTest.java | 1 + .../test/java/io/grpc/internal/InternalSubchannelTest.java | 1 + core/src/test/java/io/grpc/internal/JsonParserTest.java | 3 ++- .../java/io/grpc/internal/ManagedChannelImplBuilderTest.java | 1 + .../io/grpc/internal/ManagedChannelServiceConfigTest.java | 3 ++- core/src/test/java/io/grpc/internal/MessageDeframerTest.java | 1 + core/src/test/java/io/grpc/internal/ServerCallImplTest.java | 1 + core/src/test/java/io/grpc/internal/ServerImplTest.java | 1 + .../java/io/grpc/util/GracefulSwitchLoadBalancerTest.java | 1 + .../src/test/java/io/grpc/netty/NettyChannelBuilderTest.java | 1 + .../src/test/java/io/grpc/netty/NettyServerBuilderTest.java | 1 + .../src/test/java/io/grpc/netty/ProtocolNegotiatorsTest.java | 1 + .../test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java | 1 + .../java/io/grpc/okhttp/OkHttpProtocolNegotiatorTest.java | 1 + okhttp/src/test/java/io/grpc/okhttp/UtilsTest.java | 1 + .../test/java/io/grpc/protobuf/lite/ProtoLiteUtilsTest.java | 1 + .../src/test/java/io/grpc/testing/GrpcCleanupRuleTest.java | 1 + xds/src/test/java/io/grpc/xds/BootstrapperTest.java | 1 + .../java/io/grpc/xds/PriorityLoadBalancerProviderTest.java | 1 + xds/src/test/java/io/grpc/xds/WeightedRandomPickerTest.java | 1 + xds/src/test/java/io/grpc/xds/XdsClientImplTest.java | 1 + xds/src/test/java/io/grpc/xds/XdsClientTest.java | 1 + .../io/grpc/xds/internal/rbac/engine/AuthzEngineTest.java | 1 + 34 files changed, 37 insertions(+), 5 deletions(-) diff --git a/api/src/test/java/io/grpc/ContextsTest.java b/api/src/test/java/io/grpc/ContextsTest.java index 185100685c8..cfcfa7d4981 100644 --- a/api/src/test/java/io/grpc/ContextsTest.java +++ b/api/src/test/java/io/grpc/ContextsTest.java @@ -16,15 +16,14 @@ package io.grpc; +import static com.google.common.truth.Truth.assertThat; import static io.grpc.Contexts.interceptCall; import static io.grpc.Contexts.statusFromCancelled; -import static org.hamcrest.core.IsInstanceOf.instanceOf; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -240,7 +239,7 @@ class MockScheduledExecutorService extends ForwardingScheduledExecutorService { executorService.command.run(); assertTrue(cancellableContext.isCancelled()); - assertThat(cancellableContext.cancellationCause(), instanceOf(TimeoutException.class)); + assertThat(cancellableContext.cancellationCause()).isInstanceOf(TimeoutException.class); Status status = statusFromCancelled(cancellableContext); assertNotNull(status); diff --git a/api/src/test/java/io/grpc/MetadataTest.java b/api/src/test/java/io/grpc/MetadataTest.java index 122a11f85e5..51aecc638e7 100644 --- a/api/src/test/java/io/grpc/MetadataTest.java +++ b/api/src/test/java/io/grpc/MetadataTest.java @@ -49,6 +49,7 @@ @RunWith(JUnit4.class) public class MetadataTest { + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); private static final Metadata.BinaryMarshaller FISH_MARSHALLER = diff --git a/api/src/test/java/io/grpc/MethodDescriptorTest.java b/api/src/test/java/io/grpc/MethodDescriptorTest.java index ec89976a016..5e742fb47ed 100644 --- a/api/src/test/java/io/grpc/MethodDescriptorTest.java +++ b/api/src/test/java/io/grpc/MethodDescriptorTest.java @@ -37,6 +37,7 @@ */ @RunWith(JUnit4.class) public class MethodDescriptorTest { + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); diff --git a/api/src/test/java/io/grpc/ServerInterceptorsTest.java b/api/src/test/java/io/grpc/ServerInterceptorsTest.java index 8ea2801a2d4..fa4e7e747a2 100644 --- a/api/src/test/java/io/grpc/ServerInterceptorsTest.java +++ b/api/src/test/java/io/grpc/ServerInterceptorsTest.java @@ -56,6 +56,7 @@ public class ServerInterceptorsTest { @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); diff --git a/api/src/test/java/io/grpc/ServerServiceDefinitionTest.java b/api/src/test/java/io/grpc/ServerServiceDefinitionTest.java index 3078d44a6e7..6a84d640d78 100644 --- a/api/src/test/java/io/grpc/ServerServiceDefinitionTest.java +++ b/api/src/test/java/io/grpc/ServerServiceDefinitionTest.java @@ -52,6 +52,7 @@ public class ServerServiceDefinitionTest { = ServerMethodDefinition.create(method1, methodHandler1); private ServerMethodDefinition methodDef2 = ServerMethodDefinition.create(method2, methodHandler2); + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public ExpectedException thrown = ExpectedException.none(); diff --git a/api/src/test/java/io/grpc/ServiceDescriptorTest.java b/api/src/test/java/io/grpc/ServiceDescriptorTest.java index 0ba64e3676c..a05858680d5 100644 --- a/api/src/test/java/io/grpc/ServiceDescriptorTest.java +++ b/api/src/test/java/io/grpc/ServiceDescriptorTest.java @@ -36,6 +36,7 @@ @RunWith(JUnit4.class) public class ServiceDescriptorTest { + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); diff --git a/core/src/test/java/io/grpc/internal/AbstractClientStreamTest.java b/core/src/test/java/io/grpc/internal/AbstractClientStreamTest.java index 06db8f2869a..4ce8a467d9f 100644 --- a/core/src/test/java/io/grpc/internal/AbstractClientStreamTest.java +++ b/core/src/test/java/io/grpc/internal/AbstractClientStreamTest.java @@ -75,6 +75,7 @@ public class AbstractClientStreamTest { @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); private final StatsTraceContext statsTraceCtx = StatsTraceContext.NOOP; diff --git a/core/src/test/java/io/grpc/internal/AbstractServerStreamTest.java b/core/src/test/java/io/grpc/internal/AbstractServerStreamTest.java index 1d928d24858..65fc89be231 100644 --- a/core/src/test/java/io/grpc/internal/AbstractServerStreamTest.java +++ b/core/src/test/java/io/grpc/internal/AbstractServerStreamTest.java @@ -57,6 +57,7 @@ public class AbstractServerStreamTest { private static final int TIMEOUT_MS = 1000; private static final int MAX_MESSAGE_SIZE = 100; + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); private final WritableBufferAllocator allocator = new WritableBufferAllocator() { diff --git a/core/src/test/java/io/grpc/internal/AbstractTransportTest.java b/core/src/test/java/io/grpc/internal/AbstractTransportTest.java index aa9f269de20..5c2b9c56914 100644 --- a/core/src/test/java/io/grpc/internal/AbstractTransportTest.java +++ b/core/src/test/java/io/grpc/internal/AbstractTransportTest.java @@ -224,6 +224,7 @@ public ServerStreamTracer newServerStreamTracer(String fullMethodName, Metadata } })); + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public ExpectedException thrown = ExpectedException.none(); diff --git a/core/src/test/java/io/grpc/internal/ConnectivityStateManagerTest.java b/core/src/test/java/io/grpc/internal/ConnectivityStateManagerTest.java index e12f9bf16cb..2a759a4f386 100644 --- a/core/src/test/java/io/grpc/internal/ConnectivityStateManagerTest.java +++ b/core/src/test/java/io/grpc/internal/ConnectivityStateManagerTest.java @@ -38,6 +38,7 @@ */ @RunWith(JUnit4.class) public class ConnectivityStateManagerTest { + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); diff --git a/core/src/test/java/io/grpc/internal/DnsNameResolverTest.java b/core/src/test/java/io/grpc/internal/DnsNameResolverTest.java index 09d1df69216..c7c00994cfd 100644 --- a/core/src/test/java/io/grpc/internal/DnsNameResolverTest.java +++ b/core/src/test/java/io/grpc/internal/DnsNameResolverTest.java @@ -98,6 +98,7 @@ public class DnsNameResolverTest { @Rule public final TestRule globalTimeout = new DisableOnDebug(Timeout.seconds(10)); @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); private final Map serviceConfig = new LinkedHashMap<>(); diff --git a/core/src/test/java/io/grpc/internal/GrpcUtilTest.java b/core/src/test/java/io/grpc/internal/GrpcUtilTest.java index 30573f396d7..7a3808de6e3 100644 --- a/core/src/test/java/io/grpc/internal/GrpcUtilTest.java +++ b/core/src/test/java/io/grpc/internal/GrpcUtilTest.java @@ -44,6 +44,7 @@ @RunWith(JUnit4.class) public class GrpcUtilTest { + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); @Test diff --git a/core/src/test/java/io/grpc/internal/InternalSubchannelTest.java b/core/src/test/java/io/grpc/internal/InternalSubchannelTest.java index 37099995835..0359dbef95b 100644 --- a/core/src/test/java/io/grpc/internal/InternalSubchannelTest.java +++ b/core/src/test/java/io/grpc/internal/InternalSubchannelTest.java @@ -77,6 +77,7 @@ public class InternalSubchannelTest { @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); diff --git a/core/src/test/java/io/grpc/internal/JsonParserTest.java b/core/src/test/java/io/grpc/internal/JsonParserTest.java index d783a650590..1e74c753d4d 100644 --- a/core/src/test/java/io/grpc/internal/JsonParserTest.java +++ b/core/src/test/java/io/grpc/internal/JsonParserTest.java @@ -35,6 +35,7 @@ @RunWith(JUnit4.class) public class JsonParserTest { + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); @@ -122,4 +123,4 @@ public void objectStringName() throws IOException { assertEquals(expected, JsonParser.parse("{\"hi\": 2}")); } -} \ No newline at end of file +} diff --git a/core/src/test/java/io/grpc/internal/ManagedChannelImplBuilderTest.java b/core/src/test/java/io/grpc/internal/ManagedChannelImplBuilderTest.java index dfe9d1973d5..9534c16fbb6 100644 --- a/core/src/test/java/io/grpc/internal/ManagedChannelImplBuilderTest.java +++ b/core/src/test/java/io/grpc/internal/ManagedChannelImplBuilderTest.java @@ -80,6 +80,7 @@ public ClientCall interceptCall( }; @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); @Rule public final GrpcCleanupRule grpcCleanupRule = new GrpcCleanupRule(); diff --git a/core/src/test/java/io/grpc/internal/ManagedChannelServiceConfigTest.java b/core/src/test/java/io/grpc/internal/ManagedChannelServiceConfigTest.java index dde3adf84d6..45071504331 100644 --- a/core/src/test/java/io/grpc/internal/ManagedChannelServiceConfigTest.java +++ b/core/src/test/java/io/grpc/internal/ManagedChannelServiceConfigTest.java @@ -41,6 +41,7 @@ @RunWith(JUnit4.class) public class ManagedChannelServiceConfigTest { + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); @@ -224,4 +225,4 @@ public void getDefaultConfigSelectorFromConfig() { private static Map parseConfig(String json) throws Exception { return (Map) JsonParser.parse(json); } -} \ No newline at end of file +} diff --git a/core/src/test/java/io/grpc/internal/MessageDeframerTest.java b/core/src/test/java/io/grpc/internal/MessageDeframerTest.java index 961e51771b8..c1907e51703 100644 --- a/core/src/test/java/io/grpc/internal/MessageDeframerTest.java +++ b/core/src/test/java/io/grpc/internal/MessageDeframerTest.java @@ -337,6 +337,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable { @RunWith(JUnit4.class) public static class SizeEnforcingInputStreamTests { + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); diff --git a/core/src/test/java/io/grpc/internal/ServerCallImplTest.java b/core/src/test/java/io/grpc/internal/ServerCallImplTest.java index a4c0da2d69d..5130bc05aa7 100644 --- a/core/src/test/java/io/grpc/internal/ServerCallImplTest.java +++ b/core/src/test/java/io/grpc/internal/ServerCallImplTest.java @@ -59,6 +59,7 @@ @RunWith(JUnit4.class) public class ServerCallImplTest { + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); @Mock private ServerStream stream; @Mock private ServerCall.Listener callListener; diff --git a/core/src/test/java/io/grpc/internal/ServerImplTest.java b/core/src/test/java/io/grpc/internal/ServerImplTest.java index cf71a8b7afd..5ca3685ee87 100644 --- a/core/src/test/java/io/grpc/internal/ServerImplTest.java +++ b/core/src/test/java/io/grpc/internal/ServerImplTest.java @@ -139,6 +139,7 @@ public boolean shouldAccept(Runnable runnable) { }; private static final String AUTHORITY = "some_authority"; + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); @BeforeClass diff --git a/core/src/test/java/io/grpc/util/GracefulSwitchLoadBalancerTest.java b/core/src/test/java/io/grpc/util/GracefulSwitchLoadBalancerTest.java index f55d93c2057..2131cd725b6 100644 --- a/core/src/test/java/io/grpc/util/GracefulSwitchLoadBalancerTest.java +++ b/core/src/test/java/io/grpc/util/GracefulSwitchLoadBalancerTest.java @@ -62,6 +62,7 @@ */ @RunWith(JUnit4.class) public class GracefulSwitchLoadBalancerTest { + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); diff --git a/netty/src/test/java/io/grpc/netty/NettyChannelBuilderTest.java b/netty/src/test/java/io/grpc/netty/NettyChannelBuilderTest.java index 94fc74f5776..a4184eb47d2 100644 --- a/netty/src/test/java/io/grpc/netty/NettyChannelBuilderTest.java +++ b/netty/src/test/java/io/grpc/netty/NettyChannelBuilderTest.java @@ -41,6 +41,7 @@ @RunWith(JUnit4.class) public class NettyChannelBuilderTest { + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); private final SslContext noSslContext = null; diff --git a/netty/src/test/java/io/grpc/netty/NettyServerBuilderTest.java b/netty/src/test/java/io/grpc/netty/NettyServerBuilderTest.java index 660cd883998..44e13d80c3d 100644 --- a/netty/src/test/java/io/grpc/netty/NettyServerBuilderTest.java +++ b/netty/src/test/java/io/grpc/netty/NettyServerBuilderTest.java @@ -40,6 +40,7 @@ @RunWith(JUnit4.class) public class NettyServerBuilderTest { + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); private NettyServerBuilder builder = NettyServerBuilder.forPort(8080); diff --git a/netty/src/test/java/io/grpc/netty/ProtocolNegotiatorsTest.java b/netty/src/test/java/io/grpc/netty/ProtocolNegotiatorsTest.java index 92b95eefa25..454632715a4 100644 --- a/netty/src/test/java/io/grpc/netty/ProtocolNegotiatorsTest.java +++ b/netty/src/test/java/io/grpc/netty/ProtocolNegotiatorsTest.java @@ -122,6 +122,7 @@ public class ProtocolNegotiatorsTest { private static final int TIMEOUT_SECONDS = 60; @Rule public final TestRule globalTimeout = new DisableOnDebug(Timeout.seconds(TIMEOUT_SECONDS)); + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); private final EventLoopGroup group = new DefaultEventLoop(); diff --git a/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java b/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java index 1316463f7a3..e15bca3da09 100644 --- a/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java +++ b/okhttp/src/test/java/io/grpc/okhttp/OkHttpChannelBuilderTest.java @@ -47,6 +47,7 @@ @RunWith(JUnit4.class) public class OkHttpChannelBuilderTest { + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); @Rule public final GrpcCleanupRule grpcCleanupRule = new GrpcCleanupRule(); diff --git a/okhttp/src/test/java/io/grpc/okhttp/OkHttpProtocolNegotiatorTest.java b/okhttp/src/test/java/io/grpc/okhttp/OkHttpProtocolNegotiatorTest.java index fefa598fdd6..3a4a21b2467 100644 --- a/okhttp/src/test/java/io/grpc/okhttp/OkHttpProtocolNegotiatorTest.java +++ b/okhttp/src/test/java/io/grpc/okhttp/OkHttpProtocolNegotiatorTest.java @@ -49,6 +49,7 @@ */ @RunWith(JUnit4.class) public class OkHttpProtocolNegotiatorTest { + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); private final SSLSocket sock = mock(SSLSocket.class); diff --git a/okhttp/src/test/java/io/grpc/okhttp/UtilsTest.java b/okhttp/src/test/java/io/grpc/okhttp/UtilsTest.java index cdaf98c5736..895ba7ff7c7 100644 --- a/okhttp/src/test/java/io/grpc/okhttp/UtilsTest.java +++ b/okhttp/src/test/java/io/grpc/okhttp/UtilsTest.java @@ -37,6 +37,7 @@ @RunWith(JUnit4.class) public class UtilsTest { + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); diff --git a/protobuf-lite/src/test/java/io/grpc/protobuf/lite/ProtoLiteUtilsTest.java b/protobuf-lite/src/test/java/io/grpc/protobuf/lite/ProtoLiteUtilsTest.java index d05e884105e..6ea836f96a7 100644 --- a/protobuf-lite/src/test/java/io/grpc/protobuf/lite/ProtoLiteUtilsTest.java +++ b/protobuf-lite/src/test/java/io/grpc/protobuf/lite/ProtoLiteUtilsTest.java @@ -51,6 +51,7 @@ @RunWith(JUnit4.class) public class ProtoLiteUtilsTest { + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); private Marshaller marshaller = ProtoLiteUtils.marshaller(Type.getDefaultInstance()); diff --git a/testing/src/test/java/io/grpc/testing/GrpcCleanupRuleTest.java b/testing/src/test/java/io/grpc/testing/GrpcCleanupRuleTest.java index 1ec096522d6..3be53ae8b8b 100644 --- a/testing/src/test/java/io/grpc/testing/GrpcCleanupRuleTest.java +++ b/testing/src/test/java/io/grpc/testing/GrpcCleanupRuleTest.java @@ -51,6 +51,7 @@ public class GrpcCleanupRuleTest { public static final FakeClock fakeClock = new FakeClock(); + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public ExpectedException thrown = ExpectedException.none(); diff --git a/xds/src/test/java/io/grpc/xds/BootstrapperTest.java b/xds/src/test/java/io/grpc/xds/BootstrapperTest.java index c49e3fe964b..f664a8d01c7 100644 --- a/xds/src/test/java/io/grpc/xds/BootstrapperTest.java +++ b/xds/src/test/java/io/grpc/xds/BootstrapperTest.java @@ -40,6 +40,7 @@ @RunWith(JUnit4.class) public class BootstrapperTest { + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public ExpectedException thrown = ExpectedException.none(); @Test diff --git a/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerProviderTest.java b/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerProviderTest.java index f0d6958a5b4..a7abcf822e5 100644 --- a/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/PriorityLoadBalancerProviderTest.java @@ -34,6 +34,7 @@ /** Tests for {@link PriorityLoadBalancerProvider}. */ @RunWith(JUnit4.class) public class PriorityLoadBalancerProviderTest { + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); @SuppressWarnings("ExpectedExceptionChecker") diff --git a/xds/src/test/java/io/grpc/xds/WeightedRandomPickerTest.java b/xds/src/test/java/io/grpc/xds/WeightedRandomPickerTest.java index e88f9cbfbab..8418e80bf2b 100644 --- a/xds/src/test/java/io/grpc/xds/WeightedRandomPickerTest.java +++ b/xds/src/test/java/io/grpc/xds/WeightedRandomPickerTest.java @@ -42,6 +42,7 @@ */ @RunWith(JUnit4.class) public class WeightedRandomPickerTest { + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java index c0f48873edf..ae9e94a1e5b 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTest.java @@ -176,6 +176,7 @@ public boolean shouldAccept(Runnable command) { @Rule public final GrpcCleanupRule cleanupRule = new GrpcCleanupRule(); + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public ExpectedException thrown = ExpectedException.none(); diff --git a/xds/src/test/java/io/grpc/xds/XdsClientTest.java b/xds/src/test/java/io/grpc/xds/XdsClientTest.java index 56fca6dc09b..a7bfc166db5 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientTest.java @@ -34,6 +34,7 @@ */ @RunWith(JUnit4.class) public class XdsClientTest { + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); diff --git a/xds/src/test/java/io/grpc/xds/internal/rbac/engine/AuthzEngineTest.java b/xds/src/test/java/io/grpc/xds/internal/rbac/engine/AuthzEngineTest.java index 9716260af28..9d890430ba7 100644 --- a/xds/src/test/java/io/grpc/xds/internal/rbac/engine/AuthzEngineTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/rbac/engine/AuthzEngineTest.java @@ -44,6 +44,7 @@ /** Unit tests for constructor of CEL-based Authorization Engine. */ @RunWith(JUnit4.class) public class AuthzEngineTest { + @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); From 00e2d717a2d0a8c898357a5894c0f614dda50cb6 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Mon, 21 Sep 2020 15:52:50 -0700 Subject: [PATCH 74/86] netty: BDP ping accounting should occur after flow control It's hoped that this resolves the "too_many_pings" issue some users are seeing that is worked around by GRPC_EXPERIMENTAL_AUTOFLOWCONTROL=false. This change also avoids resetting the ping count for empty data frames (which shouldn't really happen with gRPC). The previous code failed to reset the ping count on HEADERS and WINDOW_UPDATE. The code _appeared_ to have callbacks for WINDOW_UPDATE, but was layered above the Http2Connection so was never called. Thus, this version is much more aggressive then the previous version while also addressing the correctness issue. --- .../io/grpc/netty/AbstractNettyHandler.java | 54 +++---- .../java/io/grpc/netty/ListeningEncoder.java | 136 ------------------ .../io/grpc/netty/NettyClientHandler.java | 74 +++++++++- .../io/grpc/netty/NettyServerHandler.java | 6 +- .../io/grpc/netty/NettyClientHandlerTest.java | 62 ++++++++ .../io/grpc/netty/NettyHandlerTestBase.java | 2 +- 6 files changed, 155 insertions(+), 179 deletions(-) delete mode 100644 netty/src/main/java/io/grpc/netty/ListeningEncoder.java diff --git a/netty/src/main/java/io/grpc/netty/AbstractNettyHandler.java b/netty/src/main/java/io/grpc/netty/AbstractNettyHandler.java index 6fcd03fdc42..4ab88f9efcb 100644 --- a/netty/src/main/java/io/grpc/netty/AbstractNettyHandler.java +++ b/netty/src/main/java/io/grpc/netty/AbstractNettyHandler.java @@ -16,12 +16,10 @@ package io.grpc.netty; -import static com.google.common.base.Preconditions.checkArgument; import static io.netty.handler.codec.http2.Http2CodecUtil.getEmbeddedHttp2Exception; import com.google.common.annotations.VisibleForTesting; -import io.grpc.netty.ListeningEncoder.Http2OutboundFrameListener; -import io.netty.buffer.ByteBuf; +import com.google.common.base.Preconditions; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelPromise; import io.netty.handler.codec.http2.Http2ConnectionDecoder; @@ -38,11 +36,9 @@ */ abstract class AbstractNettyHandler extends GrpcHttp2ConnectionHandler { private static final long GRACEFUL_SHUTDOWN_NO_TIMEOUT = -1; - private static final int MAX_ALLOWED_PING = 2; private final int initialConnectionWindow; - private final PingCountingListener pingCountingListener = new PingCountingListener(); - private final FlowControlPinger flowControlPing = new FlowControlPinger(MAX_ALLOWED_PING); + private final FlowControlPinger flowControlPing; private boolean autoTuneFlowControlOn; private ChannelHandlerContext ctx; @@ -55,7 +51,8 @@ abstract class AbstractNettyHandler extends GrpcHttp2ConnectionHandler { Http2ConnectionDecoder decoder, Http2ConnectionEncoder encoder, Http2Settings initialSettings, - boolean autoFlowControl) { + boolean autoFlowControl, + PingLimiter pingLimiter) { super(channelUnused, decoder, encoder, initialSettings); // During a graceful shutdown, wait until all streams are closed. @@ -65,9 +62,10 @@ abstract class AbstractNettyHandler extends GrpcHttp2ConnectionHandler { this.initialConnectionWindow = initialSettings.initialWindowSize() == null ? -1 : initialSettings.initialWindowSize(); this.autoTuneFlowControlOn = autoFlowControl; - if (encoder instanceof ListeningEncoder) { - ((ListeningEncoder) encoder).setListener(pingCountingListener); + if (pingLimiter == null) { + pingLimiter = new AllowPingLimiter(); } + this.flowControlPing = new FlowControlPinger(pingLimiter); } @Override @@ -131,7 +129,8 @@ void setAutoTuneFlowControl(boolean isOn) { final class FlowControlPinger { private static final int MAX_WINDOW_SIZE = 8 * 1024 * 1024; - private final int maxAllowedPing; + + private final PingLimiter pingLimiter; private int pingCount; private int pingReturn; private boolean pinging; @@ -139,9 +138,9 @@ final class FlowControlPinger { private float lastBandwidth; // bytes per second private long lastPingTime; - public FlowControlPinger(int maxAllowedPing) { - checkArgument(maxAllowedPing > 0, "maxAllowedPing must be positive"); - this.maxAllowedPing = maxAllowedPing; + public FlowControlPinger(PingLimiter pingLimiter) { + Preconditions.checkNotNull(pingLimiter, "pingLimiter"); + this.pingLimiter = pingLimiter; } public long payload() { @@ -156,7 +155,7 @@ public void onDataRead(int dataLength, int paddingLength) { if (!autoTuneFlowControlOn) { return; } - if (!isPinging() && pingCountingListener.pingCount < maxAllowedPing) { + if (!isPinging() && pingLimiter.isPingAllowed()) { setPinging(true); sendPing(ctx()); } @@ -235,27 +234,14 @@ void setDataSizeAndSincePing(int dataSize) { } } - private static class PingCountingListener extends Http2OutboundFrameListener { - int pingCount = 0; - - @Override - public void onWindowUpdate(int streamId, int windowSizeIncrement) { - pingCount = 0; - super.onWindowUpdate(streamId, windowSizeIncrement); - } - - @Override - public void onPing(boolean ack, long data) { - if (!ack) { - pingCount++; - } - super.onPing(ack, data); - } + /** Controls whether PINGs like those for BDP are permitted to be sent at the current time. */ + public interface PingLimiter { + public boolean isPingAllowed(); + } - @Override - public void onData(int streamId, ByteBuf data, int padding, boolean endStream) { - pingCount = 0; - super.onData(streamId, data, padding, endStream); + private static final class AllowPingLimiter implements PingLimiter { + @Override public boolean isPingAllowed() { + return true; } } } diff --git a/netty/src/main/java/io/grpc/netty/ListeningEncoder.java b/netty/src/main/java/io/grpc/netty/ListeningEncoder.java deleted file mode 100644 index 03270d399ae..00000000000 --- a/netty/src/main/java/io/grpc/netty/ListeningEncoder.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright 2020 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.netty; - -import static com.google.common.base.Preconditions.checkNotNull; - -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelPromise; -import io.netty.handler.codec.http2.DefaultHttp2ConnectionEncoder; -import io.netty.handler.codec.http2.Http2Connection; -import io.netty.handler.codec.http2.Http2ConnectionEncoder; -import io.netty.handler.codec.http2.Http2FrameWriter; -import io.netty.handler.codec.http2.StreamBufferingEncoder; - -/** A ListeningEncoder notifies {@link Http2OutboundFrameListener} on http2 outbound frame event. */ -interface ListeningEncoder { - - void setListener(Http2OutboundFrameListener listener); - - /** - * Partial implementation of (Listening subset of event) event listener for outbound http2 - * frames. - */ - class Http2OutboundFrameListener { - - /** Notifies on outbound WINDOW_UPDATE frame. */ - public void onWindowUpdate(int streamId, int windowSizeIncrement) {} - - /** Notifies on outbound PING frame. */ - public void onPing(boolean ack, long data) {} - - /** Notifies on outbound DATA frame. */ - public void onData(int streamId, ByteBuf data, int padding, boolean endStream) {} - } - - /** A {@link StreamBufferingEncoder} notifies http2 outbound frame event. */ - final class ListeningStreamBufferingEncoder - extends StreamBufferingEncoder implements ListeningEncoder { - - private Http2OutboundFrameListener listener = new Http2OutboundFrameListener(); - - public ListeningStreamBufferingEncoder(Http2ConnectionEncoder encoder) { - super(encoder); - } - - @Override - public void setListener(Http2OutboundFrameListener listener) { - this.listener = checkNotNull(listener, "listener"); - } - - @Override - public ChannelFuture writePing( - ChannelHandlerContext ctx, boolean ack, long data, ChannelPromise promise) { - listener.onPing(ack, data); - return super.writePing(ctx, ack, data, promise); - } - - @Override - public ChannelFuture writeWindowUpdate( - ChannelHandlerContext ctx, int streamId, int windowSizeIncrement, ChannelPromise promise) { - listener.onWindowUpdate(streamId, windowSizeIncrement); - return super.writeWindowUpdate(ctx, streamId, windowSizeIncrement, promise); - } - - @Override - public ChannelFuture writeData( - ChannelHandlerContext ctx, - int streamId, - ByteBuf data, - int padding, - boolean eos, - ChannelPromise promise) { - listener.onData(streamId, data, padding, eos); - return super.writeData(ctx, streamId, data, padding, eos, promise); - } - } - - /** A {@link DefaultHttp2ConnectionEncoder} notifies http2 outbound frame event. */ - final class ListeningDefaultHttp2ConnectionEncoder - extends DefaultHttp2ConnectionEncoder implements ListeningEncoder { - - private Http2OutboundFrameListener listener = new Http2OutboundFrameListener(); - - public ListeningDefaultHttp2ConnectionEncoder( - Http2Connection connection, Http2FrameWriter frameWriter) { - super(connection, frameWriter); - } - - @Override - public void setListener(Http2OutboundFrameListener listener) { - this.listener = checkNotNull(listener, "listener"); - } - - @Override - public ChannelFuture writePing( - ChannelHandlerContext ctx, boolean ack, long data, ChannelPromise promise) { - listener.onPing(ack, data); - return super.writePing(ctx, ack, data, promise); - } - - @Override - public ChannelFuture writeWindowUpdate( - ChannelHandlerContext ctx, int streamId, int windowSizeIncrement, ChannelPromise promise) { - listener.onWindowUpdate(streamId, windowSizeIncrement); - return super.writeWindowUpdate(ctx, streamId, windowSizeIncrement, promise); - } - - @Override - public ChannelFuture writeData( - ChannelHandlerContext ctx, - int streamId, - ByteBuf data, - int padding, - boolean eos, - ChannelPromise promise) { - listener.onData(streamId, data, padding, eos); - return super.writeData(ctx, streamId, data, padding, eos, promise); - } - } -} diff --git a/netty/src/main/java/io/grpc/netty/NettyClientHandler.java b/netty/src/main/java/io/grpc/netty/NettyClientHandler.java index f02b885c2b0..61052bd190c 100644 --- a/netty/src/main/java/io/grpc/netty/NettyClientHandler.java +++ b/netty/src/main/java/io/grpc/netty/NettyClientHandler.java @@ -38,7 +38,6 @@ import io.grpc.internal.KeepAliveManager; import io.grpc.internal.TransportTracer; import io.grpc.netty.GrpcHttp2HeadersUtils.GrpcHttp2ClientHeadersDecoder; -import io.grpc.netty.ListeningEncoder.ListeningStreamBufferingEncoder; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufUtil; import io.netty.buffer.Unpooled; @@ -47,6 +46,7 @@ import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelPromise; +import io.netty.handler.codec.http2.DecoratingHttp2FrameWriter; import io.netty.handler.codec.http2.DefaultHttp2Connection; import io.netty.handler.codec.http2.DefaultHttp2ConnectionDecoder; import io.netty.handler.codec.http2.DefaultHttp2ConnectionEncoder; @@ -197,8 +197,11 @@ static NettyClientHandler newHandler( frameReader = new Http2InboundFrameLogger(frameReader, frameLogger); frameWriter = new Http2OutboundFrameLogger(frameWriter, frameLogger); + PingCountingFrameWriter pingCounter; + frameWriter = pingCounter = new PingCountingFrameWriter(frameWriter); + StreamBufferingEncoder encoder = - new ListeningStreamBufferingEncoder( + new StreamBufferingEncoder( new DefaultHttp2ConnectionEncoder(connection, frameWriter)); // Create the local flow controller configured to auto-refill the connection window. @@ -237,7 +240,8 @@ public TransportTracer.FlowControlWindows read() { transportTracer, eagAttributes, authority, - autoFlowControl); + autoFlowControl, + pingCounter); } private NettyClientHandler( @@ -251,8 +255,9 @@ private NettyClientHandler( TransportTracer transportTracer, Attributes eagAttributes, String authority, - boolean autoFlowControl) { - super(/* channelUnused= */ null, decoder, encoder, settings, autoFlowControl); + boolean autoFlowControl, + PingLimiter pingLimiter) { + super(/* channelUnused= */ null, decoder, encoder, settings, autoFlowControl, pingLimiter); this.lifecycleManager = lifecycleManager; this.keepAliveManager = keepAliveManager; this.stopwatchFactory = stopwatchFactory; @@ -912,4 +917,63 @@ public void onPingRead(ChannelHandlerContext ctx, long data) throws Http2Excepti } } } + + private static class PingCountingFrameWriter extends DecoratingHttp2FrameWriter + implements AbstractNettyHandler.PingLimiter { + private int pingCount; + + public PingCountingFrameWriter(Http2FrameWriter delegate) { + super(delegate); + } + + @Override + public boolean isPingAllowed() { + // "3 strikes" may cause the server to complain, so we limit ourselves to 2 or below. + return pingCount < 2; + } + + @Override + public ChannelFuture writeHeaders( + ChannelHandlerContext ctx, int streamId, Http2Headers headers, + int padding, boolean endStream, ChannelPromise promise) { + pingCount = 0; + return super.writeHeaders(ctx, streamId, headers, padding, endStream, promise); + } + + @Override + public ChannelFuture writeHeaders( + ChannelHandlerContext ctx, int streamId, Http2Headers headers, + int streamDependency, short weight, boolean exclusive, + int padding, boolean endStream, ChannelPromise promise) { + pingCount = 0; + return super.writeHeaders(ctx, streamId, headers, streamDependency, weight, exclusive, + padding, endStream, promise); + } + + @Override + public ChannelFuture writeWindowUpdate( + ChannelHandlerContext ctx, int streamId, int windowSizeIncrement, ChannelPromise promise) { + pingCount = 0; + return super.writeWindowUpdate(ctx, streamId, windowSizeIncrement, promise); + } + + @Override + public ChannelFuture writePing( + ChannelHandlerContext ctx, boolean ack, long data, ChannelPromise promise) { + if (!ack) { + pingCount++; + } + return super.writePing(ctx, ack, data, promise); + } + + @Override + public ChannelFuture writeData( + ChannelHandlerContext ctx, int streamId, ByteBuf data, int padding, boolean endStream, + ChannelPromise promise) { + if (data.isReadable()) { + pingCount = 0; + } + return super.writeData(ctx, streamId, data, padding, endStream, promise); + } + } } diff --git a/netty/src/main/java/io/grpc/netty/NettyServerHandler.java b/netty/src/main/java/io/grpc/netty/NettyServerHandler.java index b40ee7bd453..14ce6ef6170 100644 --- a/netty/src/main/java/io/grpc/netty/NettyServerHandler.java +++ b/netty/src/main/java/io/grpc/netty/NettyServerHandler.java @@ -45,7 +45,6 @@ import io.grpc.internal.StatsTraceContext; import io.grpc.internal.TransportTracer; import io.grpc.netty.GrpcHttp2HeadersUtils.GrpcHttp2ServerHeadersDecoder; -import io.grpc.netty.ListeningEncoder.ListeningDefaultHttp2ConnectionEncoder; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufUtil; import io.netty.channel.ChannelFuture; @@ -55,6 +54,7 @@ import io.netty.handler.codec.http2.DecoratingHttp2FrameWriter; import io.netty.handler.codec.http2.DefaultHttp2Connection; import io.netty.handler.codec.http2.DefaultHttp2ConnectionDecoder; +import io.netty.handler.codec.http2.DefaultHttp2ConnectionEncoder; import io.netty.handler.codec.http2.DefaultHttp2FrameReader; import io.netty.handler.codec.http2.DefaultHttp2FrameWriter; import io.netty.handler.codec.http2.DefaultHttp2Headers; @@ -221,7 +221,7 @@ static NettyServerHandler newHandler( new DefaultHttp2LocalFlowController(connection, DEFAULT_WINDOW_UPDATE_RATIO, true)); frameWriter = new WriteMonitoringFrameWriter(frameWriter, keepAliveEnforcer); Http2ConnectionEncoder encoder = - new ListeningDefaultHttp2ConnectionEncoder(connection, frameWriter); + new DefaultHttp2ConnectionEncoder(connection, frameWriter); encoder = new Http2ControlFrameLimitEncoder(encoder, 10000); Http2ConnectionDecoder decoder = new DefaultHttp2ConnectionDecoder(connection, encoder, frameReader); @@ -263,7 +263,7 @@ private NettyServerHandler( long maxConnectionAgeGraceInNanos, final KeepAliveEnforcer keepAliveEnforcer, boolean autoFlowControl) { - super(channelUnused, decoder, encoder, settings, autoFlowControl); + super(channelUnused, decoder, encoder, settings, autoFlowControl, null); final MaxConnectionIdleManager maxConnectionIdleManager; if (maxConnectionIdleInNanos == MAX_CONNECTION_IDLE_NANOS_DISABLED) { diff --git a/netty/src/test/java/io/grpc/netty/NettyClientHandlerTest.java b/netty/src/test/java/io/grpc/netty/NettyClientHandlerTest.java index 17586ef9256..0ff2d981410 100644 --- a/netty/src/test/java/io/grpc/netty/NettyClientHandlerTest.java +++ b/netty/src/test/java/io/grpc/netty/NettyClientHandlerTest.java @@ -713,6 +713,68 @@ public void oustandingUserPingShouldNotInteractWithDataPing() throws Exception { assertEquals(1, transportTracer.getStats().keepAlivesSent); } + @Test + public void bdpPingAvoidsTooManyPingsOnSpecialServers() throws Exception { + // gRPC servers limit PINGs based on what they _send_. Some servers limit PINGs based on what is + // _received_. + createStream(); + handler().setAutoTuneFlowControl(true); + + Http2Headers headers = new DefaultHttp2Headers().status(STATUS_OK) + .set(CONTENT_TYPE_HEADER, CONTENT_TYPE_GRPC); + channelRead(headersFrame(3, headers)); + channelRead(dataFrame(3, false, content())); + verifyWrite().writePing(eq(ctx()), eq(false), eq(1234L), any(ChannelPromise.class)); + channelRead(pingFrame(true, 1234)); + + channelRead(dataFrame(3, false, content())); + verifyWrite(times(2)).writePing(eq(ctx()), eq(false), eq(1234L), any(ChannelPromise.class)); + channelRead(pingFrame(true, 1234)); + + channelRead(dataFrame(3, false, content())); + // No ping was sent + verifyWrite(times(2)).writePing(eq(ctx()), eq(false), eq(1234L), any(ChannelPromise.class)); + } + + @Test + public void bdpPingAllowedAfterSendingData() throws Exception { + // gRPC servers limit PINGs based on what they _send_. Some servers limit PINGs based on what is + // _received_. + flowControlWindow = 64 * 1024; + manualSetUp(); + createStream(); + handler().setAutoTuneFlowControl(true); + + ByteBuf content = Unpooled.buffer(64 * 1024 + 1024); + content.writerIndex(content.capacity()); + ChannelFuture future + = enqueue(new SendGrpcFrameCommand(streamTransportState, content, false)); + assertFalse(future.isDone()); // flow control limits send + + Http2Headers headers = new DefaultHttp2Headers().status(STATUS_OK) + .set(CONTENT_TYPE_HEADER, CONTENT_TYPE_GRPC); + channelRead(headersFrame(3, headers)); + channelRead(dataFrame(3, false, content())); + verifyWrite().writePing(eq(ctx()), eq(false), eq(1234L), any(ChannelPromise.class)); + channelRead(pingFrame(true, 1234)); + + channelRead(dataFrame(3, false, content())); + verifyWrite(times(2)).writePing(eq(ctx()), eq(false), eq(1234L), any(ChannelPromise.class)); + channelRead(pingFrame(true, 1234)); + + channelRead(dataFrame(3, false, content())); + // No ping was sent + verifyWrite(times(2)).writePing(eq(ctx()), eq(false), eq(1234L), any(ChannelPromise.class)); + + channelRead(windowUpdate(0, 2024)); + channelRead(windowUpdate(3, 2024)); + assertTrue(future.isDone()); + assertTrue(future.isSuccess()); + // But now one is sent + channelRead(dataFrame(3, false, content())); + verifyWrite(times(3)).writePing(eq(ctx()), eq(false), eq(1234L), any(ChannelPromise.class)); + } + @Override public void dataPingAckIsRecognized() throws Exception { super.dataPingAckIsRecognized(); diff --git a/netty/src/test/java/io/grpc/netty/NettyHandlerTestBase.java b/netty/src/test/java/io/grpc/netty/NettyHandlerTestBase.java index aab000b9279..04f65eed145 100644 --- a/netty/src/test/java/io/grpc/netty/NettyHandlerTestBase.java +++ b/netty/src/test/java/io/grpc/netty/NettyHandlerTestBase.java @@ -316,7 +316,7 @@ protected final ByteBuf serializeSettings(Http2Settings settings) { protected final ByteBuf windowUpdate(int streamId, int delta) { ChannelHandlerContext ctx = newMockContext(); - new DefaultHttp2FrameWriter().writeWindowUpdate(ctx, 0, delta, newPromise()); + new DefaultHttp2FrameWriter().writeWindowUpdate(ctx, streamId, delta, newPromise()); return captureWrite(ctx); } From f6c2d221e2b6c975c6cf465d68fe11ab12dabe55 Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Wed, 30 Sep 2020 15:31:09 -0700 Subject: [PATCH 75/86] rls: fix wrong synchronization for pickSubchannel() `RlsPicker.pickSubchannel()` does not run in SynchronizationContext, but it calls `CachingRlsLbClient.get()` which assumed running in SynchronizationContext. Fixed by removing `synchronizationContext.throwIfNotInThisSynchronizationContext()`. `CachingRlsLbClient.get()` is actually thread-safe in the sense it's guarded by lock, and `DataCacheEntry`'s fields are final. `ChildPolicyWrapper.picker` was not thread-safe. Fixed by making it volatile. Changed the test a bit since the old test doesn't really test things well. --- .../java/io/grpc/rls/CachingRlsLbClient.java | 56 ++--- .../io/grpc/rls/LbPolicyConfiguration.java | 68 +---- .../io/grpc/rls/CachingRlsLbClientTest.java | 9 +- .../grpc/rls/LbPolicyConfigurationTest.java | 60 ----- .../java/io/grpc/rls/RlsLoadBalancerTest.java | 237 +++++------------- 5 files changed, 96 insertions(+), 334 deletions(-) diff --git a/rls/src/main/java/io/grpc/rls/CachingRlsLbClient.java b/rls/src/main/java/io/grpc/rls/CachingRlsLbClient.java index e1a4152d896..aefbf7d9d16 100644 --- a/rls/src/main/java/io/grpc/rls/CachingRlsLbClient.java +++ b/rls/src/main/java/io/grpc/rls/CachingRlsLbClient.java @@ -453,7 +453,7 @@ final class DataCacheEntry extends CacheEntry { private final RouteLookupResponse response; private final long expireTime; private final long staleTime; - private ChildPolicyWrapper childPolicyWrapper; + private final ChildPolicyWrapper childPolicyWrapper; DataCacheEntry(RouteLookupRequest request, final RouteLookupResponse response) { super(request); @@ -467,21 +467,12 @@ final class DataCacheEntry extends CacheEntry { staleTime = now + staleAgeNanos; if (childPolicyWrapper.getPicker() != null) { - // using cached childPolicyWrapper - updateLbState(); + childPolicyWrapper.refreshState(); } else { createChildLbPolicy(); } } - private void updateLbState() { - childPolicyWrapper - .getHelper() - .updateBalancingState( - childPolicyWrapper.getConnectivityStateInfo().getState(), - childPolicyWrapper.getPicker()); - } - private void createChildLbPolicy() { ChildLoadBalancingPolicy childPolicy = lbPolicyConfig.getLoadBalancingPolicy(); LoadBalancerProvider lbProvider = childPolicy.getEffectiveLbProvider(); @@ -868,19 +859,15 @@ public PickResult pickSubchannel(PickSubchannelArgs args) { } if (response.hasData()) { ChildPolicyWrapper childPolicyWrapper = response.getChildPolicyWrapper(); - ConnectivityState connectivityState = - childPolicyWrapper.getConnectivityStateInfo().getState(); - switch (connectivityState) { - case IDLE: - case CONNECTING: - return PickResult.withNoResult(); - case READY: - return childPolicyWrapper.getPicker().pickSubchannel(args); - case TRANSIENT_FAILURE: - case SHUTDOWN: - default: - return useFallback(args); + SubchannelPicker picker = childPolicyWrapper.getPicker(); + if (picker == null) { + return PickResult.withNoResult(); + } + PickResult result = picker.pickSubchannel(args); + if (result.getStatus().isOk()) { + return result; } + return useFallback(args); } else if (response.hasError()) { return useFallback(args); } else { @@ -898,26 +885,11 @@ private PickResult useFallback(PickSubchannelArgs args) { // TODO(creamsoup) wait until lb is ready startFallbackChildPolicy(); } - switch (fallbackChildPolicyWrapper.getConnectivityStateInfo().getState()) { - case IDLE: - // fall through - case CONNECTING: - return PickResult.withNoResult(); - case TRANSIENT_FAILURE: - // fall through - case SHUTDOWN: - return - PickResult - .withError(fallbackChildPolicyWrapper.getConnectivityStateInfo().getStatus()); - case READY: - SubchannelPicker picker = fallbackChildPolicyWrapper.getPicker(); - if (picker == null) { - return PickResult.withNoResult(); - } - return picker.pickSubchannel(args); - default: - throw new AssertionError(); + SubchannelPicker picker = fallbackChildPolicyWrapper.getPicker(); + if (picker == null) { + return PickResult.withNoResult(); } + return picker.pickSubchannel(args); } private void startFallbackChildPolicy() { diff --git a/rls/src/main/java/io/grpc/rls/LbPolicyConfiguration.java b/rls/src/main/java/io/grpc/rls/LbPolicyConfiguration.java index 20515181fda..d4cc7672b45 100644 --- a/rls/src/main/java/io/grpc/rls/LbPolicyConfiguration.java +++ b/rls/src/main/java/io/grpc/rls/LbPolicyConfiguration.java @@ -23,19 +23,15 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; import io.grpc.ConnectivityState; -import io.grpc.ConnectivityStateInfo; -import io.grpc.LoadBalancer.CreateSubchannelArgs; import io.grpc.LoadBalancer.Helper; import io.grpc.LoadBalancer.Subchannel; import io.grpc.LoadBalancer.SubchannelPicker; -import io.grpc.LoadBalancer.SubchannelStateListener; import io.grpc.LoadBalancerProvider; import io.grpc.LoadBalancerRegistry; import io.grpc.internal.ObjectPool; import io.grpc.rls.ChildLoadBalancerHelper.ChildLoadBalancerHelperProvider; import io.grpc.rls.RlsProtoData.RouteLookupConfig; import io.grpc.util.ForwardingLoadBalancerHelper; -import io.grpc.util.ForwardingSubchannel; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -242,9 +238,8 @@ static final class ChildPolicyWrapper { private final String target; private final ChildPolicyReportingHelper helper; - private ConnectivityStateInfo connectivityStateInfo = - ConnectivityStateInfo.forNonError(ConnectivityState.IDLE); - private SubchannelPicker picker; + private volatile SubchannelPicker picker; + private ConnectivityState state; public ChildPolicyWrapper( String target, @@ -259,10 +254,6 @@ String getTarget() { return target; } - void setPicker(SubchannelPicker picker) { - this.picker = checkNotNull(picker, "picker"); - } - SubchannelPicker getPicker() { return picker; } @@ -271,32 +262,8 @@ ChildPolicyReportingHelper getHelper() { return helper; } - void setConnectivityStateInfo(ConnectivityStateInfo connectivityStateInfo) { - this.connectivityStateInfo = connectivityStateInfo; - } - - ConnectivityStateInfo getConnectivityStateInfo() { - return connectivityStateInfo; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ChildPolicyWrapper that = (ChildPolicyWrapper) o; - return Objects.equals(target, that.target) - && Objects.equals(helper, that.helper) - && Objects.equals(connectivityStateInfo, that.connectivityStateInfo) - && Objects.equals(picker, that.picker); - } - - @Override - public int hashCode() { - return Objects.hash(target, helper, connectivityStateInfo, picker); + void refreshState() { + helper.updateBalancingState(state, picker); } @Override @@ -304,8 +271,8 @@ public String toString() { return MoreObjects.toStringHelper(this) .add("target", target) .add("helper", helper) - .add("connectivityStateInfo", connectivityStateInfo) .add("picker", picker) + .add("state", state) .toString(); } @@ -335,32 +302,11 @@ protected Helper delegate() { @Override public void updateBalancingState(ConnectivityState newState, SubchannelPicker newPicker) { - setPicker(newPicker); + picker = newPicker; + state = newState; super.updateBalancingState(newState, newPicker); listener.onStatusChanged(newState); } - - @Override - public Subchannel createSubchannel(CreateSubchannelArgs args) { - final Subchannel subchannel = super.createSubchannel(args); - return new ForwardingSubchannel() { - @Override - protected Subchannel delegate() { - return subchannel; - } - - @Override - public void start(final SubchannelStateListener listener) { - super.start(new SubchannelStateListener() { - @Override - public void onSubchannelState(ConnectivityStateInfo newState) { - setConnectivityStateInfo(newState); - listener.onSubchannelState(newState); - } - }); - } - }; - } } } diff --git a/rls/src/test/java/io/grpc/rls/CachingRlsLbClientTest.java b/rls/src/test/java/io/grpc/rls/CachingRlsLbClientTest.java index 8d7a4febbaa..2a12e5c2c1d 100644 --- a/rls/src/test/java/io/grpc/rls/CachingRlsLbClientTest.java +++ b/rls/src/test/java/io/grpc/rls/CachingRlsLbClientTest.java @@ -418,7 +418,14 @@ public LoadBalancer newLoadBalancer(final Helper helper) { @Override public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { // TODO: make the picker accessible - helper.updateBalancingState(ConnectivityState.READY, mock(SubchannelPicker.class)); + helper.updateBalancingState( + ConnectivityState.READY, + new SubchannelPicker() { + @Override + public PickResult pickSubchannel(PickSubchannelArgs args) { + return PickResult.withSubchannel(mock(Subchannel.class)); + } + }); } @Override diff --git a/rls/src/test/java/io/grpc/rls/LbPolicyConfigurationTest.java b/rls/src/test/java/io/grpc/rls/LbPolicyConfigurationTest.java index 25fc9de29f3..7c6be039c24 100644 --- a/rls/src/test/java/io/grpc/rls/LbPolicyConfigurationTest.java +++ b/rls/src/test/java/io/grpc/rls/LbPolicyConfigurationTest.java @@ -16,25 +16,17 @@ package io.grpc.rls; -import static com.google.common.base.Preconditions.checkState; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.fail; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.grpc.Attributes; import io.grpc.ConnectivityState; -import io.grpc.ConnectivityStateInfo; -import io.grpc.EquivalentAddressGroup; -import io.grpc.LoadBalancer.CreateSubchannelArgs; import io.grpc.LoadBalancer.Helper; -import io.grpc.LoadBalancer.Subchannel; import io.grpc.LoadBalancer.SubchannelPicker; -import io.grpc.LoadBalancer.SubchannelStateListener; import io.grpc.LoadBalancerProvider; import io.grpc.LoadBalancerRegistry; import io.grpc.rls.ChildLoadBalancerHelper.ChildLoadBalancerHelperProvider; @@ -44,7 +36,6 @@ import io.grpc.rls.LbPolicyConfiguration.ChildPolicyWrapper.ChildPolicyReportingHelper; import io.grpc.rls.LbPolicyConfiguration.InvalidChildPolicyConfigException; import io.grpc.rls.LbPolicyConfiguration.RefCountedChildPolicyWrapperFactory; -import java.net.SocketAddress; import java.util.Map; import org.junit.Test; import org.junit.runner.RunWith; @@ -133,31 +124,6 @@ public void childLoadBalancingPolicy_tooManyChildPolicies() { } } - @Test - public void subchannelStateChange_updateChildPolicyWrapper() { - ChildPolicyWrapper childPolicyWrapper = factory.createOrGet("foo.google.com"); - ChildPolicyReportingHelper childPolicyReportingHelper = childPolicyWrapper.getHelper(); - FakeSubchannel fakeSubchannel = new FakeSubchannel(); - when(helper.createSubchannel(any(CreateSubchannelArgs.class))).thenReturn(fakeSubchannel); - Subchannel subchannel = - childPolicyReportingHelper - .createSubchannel( - CreateSubchannelArgs.newBuilder() - .setAddresses(new EquivalentAddressGroup(mock(SocketAddress.class))) - .build()); - subchannel.start(new SubchannelStateListener() { - @Override - public void onSubchannelState(ConnectivityStateInfo newState) { - // no-op - } - }); - - fakeSubchannel.updateState(ConnectivityStateInfo.forNonError(ConnectivityState.CONNECTING)); - - assertThat(childPolicyWrapper.getConnectivityStateInfo()) - .isEqualTo(ConnectivityStateInfo.forNonError(ConnectivityState.CONNECTING)); - } - @Test public void updateBalancingState_triggersListener() { ChildPolicyWrapper childPolicyWrapper = factory.createOrGet("foo.google.com"); @@ -171,30 +137,4 @@ public void updateBalancingState_triggersListener() { // picker governs childPickers will be reported to parent LB verify(helper).updateBalancingState(ConnectivityState.READY, picker); } - - private static class FakeSubchannel extends Subchannel { - - private SubchannelStateListener listener; - - @Override - public void start(SubchannelStateListener listener) { - this.listener = listener; - } - - void updateState(ConnectivityStateInfo newState) { - checkState(listener != null, "channel is not started yet"); - listener.onSubchannelState(newState); - } - - @Override - public void shutdown() {} - - @Override - public void requestConnection() {} - - @Override - public Attributes getAttributes() { - return null; - } - } } diff --git a/rls/src/test/java/io/grpc/rls/RlsLoadBalancerTest.java b/rls/src/test/java/io/grpc/rls/RlsLoadBalancerTest.java index 045c00c3032..b92963063b3 100644 --- a/rls/src/test/java/io/grpc/rls/RlsLoadBalancerTest.java +++ b/rls/src/test/java/io/grpc/rls/RlsLoadBalancerTest.java @@ -30,7 +30,6 @@ import com.google.common.base.Converter; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.util.concurrent.SettableFuture; import io.grpc.Attributes; import io.grpc.CallOptions; import io.grpc.ChannelLogger; @@ -51,8 +50,8 @@ import io.grpc.MethodDescriptor; import io.grpc.MethodDescriptor.Marshaller; import io.grpc.MethodDescriptor.MethodType; -import io.grpc.NameResolver.ConfigOrError; import io.grpc.NameResolver; +import io.grpc.NameResolver.ConfigOrError; import io.grpc.Status; import io.grpc.SynchronizationContext; import io.grpc.inprocess.InProcessChannelBuilder; @@ -60,7 +59,6 @@ import io.grpc.internal.JsonParser; import io.grpc.internal.PickSubchannelArgsImpl; import io.grpc.lookup.v1.RouteLookupServiceGrpc; -import io.grpc.rls.CachingRlsLbClient.RlsPicker; import io.grpc.rls.RlsLoadBalancer.CachingRlsLbClientBuilderProvider; import io.grpc.rls.RlsProtoConverters.RouteLookupResponseConverter; import io.grpc.rls.RlsProtoData.RouteLookupRequest; @@ -76,7 +74,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import javax.annotation.Nonnull; import org.junit.After; import org.junit.Before; @@ -180,140 +177,78 @@ public void tearDown() throws Exception { } @Test - public void lb_working() throws Exception { - final InOrder inOrder = inOrder(helper); - + public void lb_working() { + InOrder inOrder = inOrder(helper); inOrder.verify(helper) .updateBalancingState(eq(ConnectivityState.CONNECTING), pickerCaptor.capture()); - assertThat(pickerCaptor.getValue()).isInstanceOf(RlsPicker.class); - final RlsPicker picker = (RlsPicker) pickerCaptor.getValue(); - final Metadata headers = new Metadata(); - - blockingRunInSyncContext( - new Runnable() { - @Override - public void run() { - PickResult res = - picker.pickSubchannel( - new PickSubchannelArgsImpl(fakeSearchMethod, headers, CallOptions.DEFAULT)); - // verify pending - assertThat(res.getSubchannel()).isNull(); - assertThat(res.getStatus().isOk()).isTrue(); - } - }); - + SubchannelPicker picker = pickerCaptor.getValue(); + Metadata headers = new Metadata(); + PickResult res = picker.pickSubchannel( + new PickSubchannelArgsImpl(fakeSearchMethod, headers, CallOptions.DEFAULT)); inOrder.verify(helper).createSubchannel(any(CreateSubchannelArgs.class)); inOrder.verify(helper) - .updateBalancingState(eq(ConnectivityState.CONNECTING), pickerCaptor.capture()); - assertThat(subchannels).hasSize(1); + .updateBalancingState(eq(ConnectivityState.CONNECTING), any(SubchannelPicker.class)); inOrder.verifyNoMoreInteractions(); + assertThat(res.getStatus().isOk()).isTrue(); + assertThat(subchannelIsReady(res.getSubchannel())).isFalse(); - final FakeSubchannel searchSubchannel = subchannels.getLast(); + assertThat(subchannels).hasSize(1); + FakeSubchannel searchSubchannel = subchannels.getLast(); searchSubchannel.updateState(ConnectivityStateInfo.forNonError(ConnectivityState.READY)); inOrder.verify(helper) .updateBalancingState(eq(ConnectivityState.READY), pickerCaptor.capture()); - - assertThat(pickerCaptor.getValue()).isInstanceOf(RlsPicker.class); - final RlsPicker picker2 = (RlsPicker) pickerCaptor.getValue(); - assertThat(picker2).isEqualTo(picker); - blockingRunInSyncContext( - new Runnable() { - @Override - public void run() { - PickResult res = picker2.pickSubchannel( - new PickSubchannelArgsImpl(fakeSearchMethod, headers, CallOptions.DEFAULT)); - // verify success. Subchannel is wrapped, so checking attributes. - assertThat(res.getSubchannel()).isNotNull(); - assertThat(res.getSubchannel().getAddresses()) - .isEqualTo(searchSubchannel.getAddresses()); - assertThat(res.getSubchannel().getAttributes()) - .isEqualTo(searchSubchannel.getAttributes()); - assertThat(res.getStatus().isOk()).isTrue(); - } - }); - inOrder.verifyNoMoreInteractions(); + assertThat(subchannelIsReady(res.getSubchannel())).isTrue(); + assertThat(res.getSubchannel().getAddresses()).isEqualTo(searchSubchannel.getAddresses()); + assertThat(res.getSubchannel().getAttributes()).isEqualTo(searchSubchannel.getAttributes()); - // rescue should be pending status - blockingRunInSyncContext( - new Runnable() { - @Override - public void run() { - PickResult res = - picker.pickSubchannel( - new PickSubchannelArgsImpl(fakeRescueMethod, headers, CallOptions.DEFAULT)); - assertThat(res.getSubchannel()).isNull(); - assertThat(res.getStatus().isOk()).isTrue(); - } - }); - + // rescue should be pending status although the overall channel state is READY + res = picker.pickSubchannel( + new PickSubchannelArgsImpl(fakeRescueMethod, headers, CallOptions.DEFAULT)); inOrder.verify(helper).createSubchannel(any(CreateSubchannelArgs.class)); // other rls picker itself is ready due to first channel. inOrder.verify(helper) .updateBalancingState(eq(ConnectivityState.READY), pickerCaptor.capture()); - assertThat(subchannels).hasSize(2); inOrder.verifyNoMoreInteractions(); + assertThat(res.getStatus().isOk()).isTrue(); + assertThat(subchannelIsReady(res.getSubchannel())).isFalse(); + assertThat(subchannels).hasSize(2); + FakeSubchannel rescueSubchannel = subchannels.getLast(); - // rescue subchannel is connecting + // search subchannel is down, rescue subchannel is connecting searchSubchannel.updateState(ConnectivityStateInfo.forTransientFailure(Status.NOT_FOUND)); - inOrder.verify(helper) .updateBalancingState(eq(ConnectivityState.CONNECTING), pickerCaptor.capture()); - final FakeSubchannel rescueSubchannel = subchannels.getLast(); rescueSubchannel.updateState(ConnectivityStateInfo.forNonError(ConnectivityState.READY)); inOrder.verify(helper) .updateBalancingState(eq(ConnectivityState.READY), pickerCaptor.capture()); // search again, use pending fallback because searchSubchannel is in failure mode - blockingRunInSyncContext( - new Runnable() { - @Override - public void run() { - PickResult res = - picker.pickSubchannel( - new PickSubchannelArgsImpl(fakeSearchMethod, headers, CallOptions.DEFAULT)); - assertThat(res.getSubchannel()).isNull(); - assertThat(res.getStatus().isOk()).isTrue(); - } - }); + res = picker.pickSubchannel( + new PickSubchannelArgsImpl(fakeSearchMethod, headers, CallOptions.DEFAULT)); + assertThat(res.getStatus().isOk()).isTrue(); + assertThat(subchannelIsReady(res.getSubchannel())).isFalse(); inOrder.verify(helper).createSubchannel(any(CreateSubchannelArgs.class)); assertThat(subchannels).hasSize(3); - final FakeSubchannel fallbackSubchannel = subchannels.getLast(); + FakeSubchannel fallbackSubchannel = subchannels.getLast(); fallbackSubchannel.updateState(ConnectivityStateInfo.forNonError(ConnectivityState.READY)); inOrder.verify(helper, times(2)) .updateBalancingState(eq(ConnectivityState.READY), pickerCaptor.capture()); inOrder.verifyNoMoreInteractions(); - blockingRunInSyncContext( - new Runnable() { - @Override - public void run() { - PickResult res = - picker.pickSubchannel( - new PickSubchannelArgsImpl(fakeSearchMethod, headers, CallOptions.DEFAULT)); - assertThat(res.getSubchannel().getAddresses()) - .isEqualTo(fallbackSubchannel.getAddresses()); - assertThat(res.getSubchannel().getAttributes()) - .isEqualTo(fallbackSubchannel.getAttributes()); - assertThat(res.getStatus().isOk()).isTrue(); - } - }); - blockingRunInSyncContext( - new Runnable() { - @Override - public void run() { - PickResult res = - picker.pickSubchannel( - new PickSubchannelArgsImpl(fakeRescueMethod, headers, CallOptions.DEFAULT)); - assertThat(res.getSubchannel().getAddresses()) - .isEqualTo(rescueSubchannel.getAddresses()); - assertThat(res.getSubchannel().getAttributes()) - .isEqualTo(rescueSubchannel.getAttributes()); - assertThat(res.getStatus().isOk()).isTrue(); - } - }); + res = picker.pickSubchannel( + new PickSubchannelArgsImpl(fakeSearchMethod, headers, CallOptions.DEFAULT)); + assertThat(subchannelIsReady(res.getSubchannel())).isTrue(); + assertThat(res.getSubchannel().getAddresses()).isEqualTo(searchSubchannel.getAddresses()); + assertThat(res.getSubchannel().getAttributes()).isEqualTo(searchSubchannel.getAttributes()); + + res = picker.pickSubchannel( + new PickSubchannelArgsImpl(fakeRescueMethod, headers, CallOptions.DEFAULT)); + assertThat(subchannelIsReady(res.getSubchannel())).isTrue(); + assertThat(res.getSubchannel().getAddresses()).isEqualTo(rescueSubchannel.getAddresses()); + assertThat(res.getSubchannel().getAttributes()).isEqualTo(rescueSubchannel.getAttributes()); // all channels are failed rescueSubchannel.updateState(ConnectivityStateInfo.forTransientFailure(Status.NOT_FOUND)); @@ -326,27 +261,18 @@ public void run() { } @Test - public void lb_nameResolutionFailed() throws Exception { - final InOrder inOrder = inOrder(helper); + public void lb_nameResolutionFailed() { + InOrder inOrder = inOrder(helper); inOrder.verify(helper) .updateBalancingState(eq(ConnectivityState.CONNECTING), pickerCaptor.capture()); - assertThat(pickerCaptor.getValue()).isInstanceOf(RlsPicker.class); - final RlsPicker picker = (RlsPicker) pickerCaptor.getValue(); - final Metadata headers = new Metadata(); - - blockingRunInSyncContext( - new Runnable() { - @Override - public void run() { - PickResult res = - picker.pickSubchannel( - new PickSubchannelArgsImpl(fakeSearchMethod, headers, CallOptions.DEFAULT)); - // verify pending - assertThat(res.getSubchannel()).isNull(); - assertThat(res.getStatus().isOk()).isTrue(); - } - }); + SubchannelPicker picker = pickerCaptor.getValue(); + Metadata headers = new Metadata(); + PickResult res = + picker.pickSubchannel( + new PickSubchannelArgsImpl(fakeSearchMethod, headers, CallOptions.DEFAULT)); + assertThat(res.getStatus().isOk()).isTrue(); + assertThat(subchannelIsReady(res.getSubchannel())).isFalse(); inOrder.verify(helper).createSubchannel(any(CreateSubchannelArgs.class)); inOrder.verify(helper) @@ -354,29 +280,19 @@ public void run() { assertThat(subchannels).hasSize(1); inOrder.verifyNoMoreInteractions(); - final FakeSubchannel searchSubchannel = subchannels.getLast(); + FakeSubchannel searchSubchannel = subchannels.getLast(); searchSubchannel.updateState(ConnectivityStateInfo.forNonError(ConnectivityState.READY)); inOrder.verify(helper) .updateBalancingState(eq(ConnectivityState.READY), pickerCaptor.capture()); - assertThat(pickerCaptor.getValue()).isInstanceOf(RlsPicker.class); - final RlsPicker picker2 = (RlsPicker) pickerCaptor.getValue(); + SubchannelPicker picker2 = pickerCaptor.getValue(); assertThat(picker2).isEqualTo(picker); - blockingRunInSyncContext( - new Runnable() { - @Override - public void run() { - PickResult res = picker2.pickSubchannel( - new PickSubchannelArgsImpl(fakeSearchMethod, headers, CallOptions.DEFAULT)); - // verify success. Subchannel is wrapped, so checking attributes. - assertThat(res.getSubchannel()).isNotNull(); - assertThat(res.getSubchannel().getAddresses()) - .isEqualTo(searchSubchannel.getAddresses()); - assertThat(res.getSubchannel().getAttributes()) - .isEqualTo(searchSubchannel.getAttributes()); - assertThat(res.getStatus().isOk()).isTrue(); - } - }); + res = picker2.pickSubchannel( + new PickSubchannelArgsImpl(fakeSearchMethod, headers, CallOptions.DEFAULT)); + // verify success. Subchannel is wrapped, so checking attributes. + assertThat(subchannelIsReady(res.getSubchannel())).isTrue(); + assertThat(res.getSubchannel().getAddresses()).isEqualTo(searchSubchannel.getAddresses()); + assertThat(res.getSubchannel().getAttributes()).isEqualTo(searchSubchannel.getAttributes()); inOrder.verifyNoMoreInteractions(); @@ -384,36 +300,11 @@ public void run() { verify(helper) .updateBalancingState(eq(ConnectivityState.TRANSIENT_FAILURE), pickerCaptor.capture()); - final SubchannelPicker failedPicker = pickerCaptor.getValue(); - blockingRunInSyncContext( - new Runnable() { - @Override - public void run() { - PickResult res = failedPicker.pickSubchannel( - new PickSubchannelArgsImpl(fakeSearchMethod, headers, CallOptions.DEFAULT)); - assertThat(res.getSubchannel()).isNull(); - assertThat(res.getStatus().isOk()).isFalse(); - } - }); - } - - private void blockingRunInSyncContext(final Runnable command) throws Exception { - final SettableFuture exceptionFuture = SettableFuture.create(); - syncContext.execute(new Runnable() { - @Override - public void run() { - try { - command.run(); - exceptionFuture.set(null); - } catch (Exception e) { - exceptionFuture.set(e); - } - } - }); - Exception exception = exceptionFuture.get(5, TimeUnit.SECONDS); - if (exception != null) { - throw exception; - } + SubchannelPicker failedPicker = pickerCaptor.getValue(); + res = failedPicker.pickSubchannel( + new PickSubchannelArgsImpl(fakeSearchMethod, headers, CallOptions.DEFAULT)); + assertThat(res.getStatus().isOk()).isFalse(); + assertThat(subchannelIsReady(res.getSubchannel())).isFalse(); } @SuppressWarnings("unchecked") @@ -566,6 +457,7 @@ private static final class FakeSubchannel extends Subchannel { private final Attributes attributes; private List eags; private SubchannelStateListener listener; + private boolean isReady; public FakeSubchannel(List eags, Attributes attributes) { this.eags = Collections.unmodifiableList(eags); @@ -602,6 +494,11 @@ public void requestConnection() { public void updateState(ConnectivityStateInfo newState) { listener.onSubchannelState(newState); + isReady = newState.getState().equals(ConnectivityState.READY); } } + + private static boolean subchannelIsReady(Subchannel subchannel) { + return subchannel instanceof FakeSubchannel && ((FakeSubchannel) subchannel).isReady; + } } From 0cd56c29d61f1f09bbed7e15cd8f4b6c25cdda85 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Thu, 1 Oct 2020 15:10:30 -0500 Subject: [PATCH 76/86] stub: Only throw on cancellation for streaming responses Unary are far more common than streaming, and we're throwing for unary even though it doesn't help the service. Let's stop doing that. We also stop throwing in onComplete() for all cases, because it doesn't help any service; it doesn't stop the service's processing and isn't even all that informative since the cancellation can happen even after onComplete() is called. --- .../main/java/io/grpc/stub/ServerCalls.java | 60 ++++++++++++------- .../java/io/grpc/stub/ServerCallsTest.java | 8 +-- 2 files changed, 39 insertions(+), 29 deletions(-) diff --git a/stub/src/main/java/io/grpc/stub/ServerCalls.java b/stub/src/main/java/io/grpc/stub/ServerCalls.java index bb9720984dd..ba08139b716 100644 --- a/stub/src/main/java/io/grpc/stub/ServerCalls.java +++ b/stub/src/main/java/io/grpc/stub/ServerCalls.java @@ -48,7 +48,7 @@ private ServerCalls() { */ public static ServerCallHandler asyncUnaryCall( UnaryMethod method) { - return new UnaryServerCallHandler<>(method); + return new UnaryServerCallHandler<>(method, false); } /** @@ -58,7 +58,7 @@ public static ServerCallHandler asyncUnaryCall( */ public static ServerCallHandler asyncServerStreamingCall( ServerStreamingMethod method) { - return new UnaryServerCallHandler<>(method); + return new UnaryServerCallHandler<>(method, true); } /** @@ -68,7 +68,7 @@ public static ServerCallHandler asyncServerStreamingC */ public static ServerCallHandler asyncClientStreamingCall( ClientStreamingMethod method) { - return new StreamingServerCallHandler<>(method); + return new StreamingServerCallHandler<>(method, false); } /** @@ -78,7 +78,7 @@ public static ServerCallHandler asyncClientStreamingC */ public static ServerCallHandler asyncBidiStreamingCall( BidiStreamingMethod method) { - return new StreamingServerCallHandler<>(method); + return new StreamingServerCallHandler<>(method, true); } /** @@ -113,10 +113,12 @@ private static final class UnaryServerCallHandler implements ServerCallHandler { private final UnaryRequestMethod method; + private final boolean serverStreaming; // Non private to avoid synthetic class - UnaryServerCallHandler(UnaryRequestMethod method) { + UnaryServerCallHandler(UnaryRequestMethod method, boolean serverStreaming) { this.method = method; + this.serverStreaming = serverStreaming; } @Override @@ -125,7 +127,7 @@ public ServerCall.Listener startCall(ServerCall call, Metadat call.getMethodDescriptor().getType().clientSendsOneMessage(), "asyncUnaryRequestCall is only for clientSendsOneMessage methods"); ServerCallStreamObserverImpl responseObserver = - new ServerCallStreamObserverImpl<>(call); + new ServerCallStreamObserverImpl<>(call, serverStreaming); // We expect only 1 request, but we ask for 2 requests here so that if a misbehaving client // sends more than 1 requests, ServerCall will catch it. Note that disabling auto // inbound flow control has no effect on unary calls. @@ -189,9 +191,11 @@ public void onHalfClose() { @Override public void onCancel() { - responseObserver.cancelled = true; if (responseObserver.onCancelHandler != null) { responseObserver.onCancelHandler.run(); + } else { + // Only trigger exceptions if unable to provide notification via a callback + responseObserver.cancelled = true; } } @@ -209,16 +213,18 @@ private static final class StreamingServerCallHandler implements ServerCallHandler { private final StreamingRequestMethod method; + private final boolean bidi; // Non private to avoid synthetic class - StreamingServerCallHandler(StreamingRequestMethod method) { + StreamingServerCallHandler(StreamingRequestMethod method, boolean bidi) { this.method = method; + this.bidi = bidi; } @Override public ServerCall.Listener startCall(ServerCall call, Metadata headers) { ServerCallStreamObserverImpl responseObserver = - new ServerCallStreamObserverImpl<>(call); + new ServerCallStreamObserverImpl<>(call, bidi); StreamObserver requestObserver = method.invoke(responseObserver); responseObserver.freeze(); if (responseObserver.autoRequestEnabled) { @@ -262,14 +268,19 @@ public void onHalfClose() { @Override public void onCancel() { - responseObserver.cancelled = true; if (responseObserver.onCancelHandler != null) { responseObserver.onCancelHandler.run(); + } else { + // Only trigger exceptions if unable to provide notification via a callback. Even though + // onError would provide notification to the server, we still throw an error since there + // isn't a guaranteed callback available. If the cancellation happened in a different + // order the service could be surprised to see the exception. + responseObserver.cancelled = true; } if (!halfClosed) { requestObserver.onError( Status.CANCELLED - .withDescription("cancelled before receiving half close") + .withDescription("client cancelled") .asRuntimeException()); } } @@ -300,6 +311,7 @@ private interface StreamingRequestMethod { private static final class ServerCallStreamObserverImpl extends ServerCallStreamObserver { final ServerCall call; + private final boolean serverStreamingOrBidi; volatile boolean cancelled; private boolean frozen; private boolean autoRequestEnabled = true; @@ -310,8 +322,9 @@ private static final class ServerCallStreamObserverImpl private boolean completed = false; // Non private to avoid synthetic class - ServerCallStreamObserverImpl(ServerCall call) { + ServerCallStreamObserverImpl(ServerCall call, boolean serverStreamingOrBidi) { this.call = call; + this.serverStreamingOrBidi = serverStreamingOrBidi; } private void freeze() { @@ -331,10 +344,17 @@ public void setCompression(String compression) { @Override public void onNext(RespT response) { if (cancelled) { - if (onCancelHandler == null) { - throw Status.CANCELLED.withDescription("call already cancelled").asRuntimeException(); + if (serverStreamingOrBidi) { + throw Status.CANCELLED + .withDescription("call already cancelled. " + + "Use ServerCallStreamObserver.setOnCancelHandler() to disable this exception") + .asRuntimeException(); + } else { + // We choose not to throw for unary responses. The exception is intended to stop servers + // from continuing processing, but for unary responses there is no further processing + // so throwing an exception would not provide a benefit and would increase application + // complexity. } - return; } checkState(!aborted, "Stream was terminated by error, no further calls are allowed"); checkState(!completed, "Stream is already completed, no further calls are allowed"); @@ -357,14 +377,8 @@ public void onError(Throwable t) { @Override public void onCompleted() { - if (cancelled) { - if (onCancelHandler == null) { - throw Status.CANCELLED.withDescription("call already cancelled").asRuntimeException(); - } - } else { - call.close(Status.OK, new Metadata()); - completed = true; - } + call.close(Status.OK, new Metadata()); + completed = true; } @Override diff --git a/stub/src/test/java/io/grpc/stub/ServerCallsTest.java b/stub/src/test/java/io/grpc/stub/ServerCallsTest.java index 29981932fb7..f9ceb82166d 100644 --- a/stub/src/test/java/io/grpc/stub/ServerCallsTest.java +++ b/stub/src/test/java/io/grpc/stub/ServerCallsTest.java @@ -195,12 +195,8 @@ public StreamObserver invoke(StreamObserver responseObserver) } catch (StatusRuntimeException expected) { // Expected } - try { - callObserver.get().onCompleted(); - fail("Expected cancellation exception when onCallHandler not set"); - } catch (StatusRuntimeException expected) { - // Expected - } + // No exception + callObserver.get().onCompleted(); } @Test From ec0d01d7a40af3d7f7b5e2f4d4c549bd27701f4e Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Thu, 1 Oct 2020 09:18:26 -0700 Subject: [PATCH 77/86] netty: TCP close during TLS handshake should be UNAVAILABLE Normally the first exception/event experienced is the cause and is followed by a stampede of ClosedChannelExceptions. In this case, SslHandler is manufacturing a ClosedChannelException of its own and propagating it _before_ the trigger event. This might be considered a bug, but changing SslHandler's behavior would be very risky and almost certainly break someone's code. Fixes #7376 --- .../java/io/grpc/netty/ProtocolNegotiators.java | 14 +++++++++++++- .../io/grpc/netty/ProtocolNegotiatorsTest.java | 16 ++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/netty/src/main/java/io/grpc/netty/ProtocolNegotiators.java b/netty/src/main/java/io/grpc/netty/ProtocolNegotiators.java index 27b2c74be2d..b3ebf2367e7 100644 --- a/netty/src/main/java/io/grpc/netty/ProtocolNegotiators.java +++ b/netty/src/main/java/io/grpc/netty/ProtocolNegotiators.java @@ -58,6 +58,7 @@ import io.netty.util.AttributeMap; import java.net.SocketAddress; import java.net.URI; +import java.nio.channels.ClosedChannelException; import java.util.Arrays; import java.util.concurrent.Executor; import java.util.logging.Level; @@ -372,7 +373,18 @@ protected void userEventTriggered0(ChannelHandlerContext ctx, Object evt) throws ctx.fireExceptionCaught(ex); } } else { - ctx.fireExceptionCaught(handshakeEvent.cause()); + Throwable t = handshakeEvent.cause(); + if (t instanceof ClosedChannelException) { + // On channelInactive(), SslHandler creates its own ClosedChannelException and + // propagates it before the actual channelInactive(). So we assume here that any + // such exception is from channelInactive() and emulate the normal behavior of + // WriteBufferingAndExceptionHandler + t = Status.UNAVAILABLE + .withDescription("Connection closed while performing TLS negotiation") + .withCause(t) + .asRuntimeException(); + } + ctx.fireExceptionCaught(t); } } else { super.userEventTriggered0(ctx, evt); diff --git a/netty/src/test/java/io/grpc/netty/ProtocolNegotiatorsTest.java b/netty/src/test/java/io/grpc/netty/ProtocolNegotiatorsTest.java index 454632715a4..d83a5fabaf5 100644 --- a/netty/src/test/java/io/grpc/netty/ProtocolNegotiatorsTest.java +++ b/netty/src/test/java/io/grpc/netty/ProtocolNegotiatorsTest.java @@ -33,6 +33,8 @@ import io.grpc.Grpc; import io.grpc.InternalChannelz.Security; import io.grpc.SecurityLevel; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; import io.grpc.internal.GrpcAttributes; import io.grpc.internal.testing.TestUtils; import io.grpc.netty.ProtocolNegotiators.ClientTlsHandler; @@ -534,6 +536,20 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { assertNull(grpcHandlerCtx); } + @Test + public void clientTlsHandler_closeDuringNegotiation() throws Exception { + ClientTlsHandler handler = new ClientTlsHandler(grpcHandler, sslContext, "authority", null); + pipeline.addLast(new WriteBufferingAndExceptionHandler(handler)); + ChannelFuture pendingWrite = channel.writeAndFlush(NettyClientHandler.NOOP_MESSAGE); + + // SslHandler fires userEventTriggered() before channelInactive() + pipeline.fireChannelInactive(); + + assertThat(pendingWrite.cause()).isInstanceOf(StatusRuntimeException.class); + assertThat(Status.fromThrowable(pendingWrite.cause()).getCode()) + .isEqualTo(Status.Code.UNAVAILABLE); + } + @Test public void engineLog() { ChannelHandler handler = new ServerTlsHandler(grpcHandler, sslContext, null); From 594cc76292bdf30a97502e1536a0c48c248d1638 Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Thu, 1 Oct 2020 13:32:13 -0700 Subject: [PATCH 78/86] xds: advertise send_all_clusters client feature in LRS requests (#7477) --- xds/src/main/java/io/grpc/xds/LoadReportClient.java | 6 +++--- xds/src/test/java/io/grpc/xds/LoadReportClientTest.java | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/LoadReportClient.java b/xds/src/main/java/io/grpc/xds/LoadReportClient.java index d39db86ca16..821ca71d986 100644 --- a/xds/src/main/java/io/grpc/xds/LoadReportClient.java +++ b/xds/src/main/java/io/grpc/xds/LoadReportClient.java @@ -84,9 +84,9 @@ final class LoadReportClient { this.syncContext = checkNotNull(syncContext, "syncContext"); this.timerService = checkNotNull(scheduledExecutorService, "timeService"); this.backoffPolicyProvider = checkNotNull(backoffPolicyProvider, "backoffPolicyProvider"); - checkNotNull(stopwatchSupplier, "stopwatchSupplier"); - this.retryStopwatch = stopwatchSupplier.get(); - this.node = checkNotNull(node, "node"); + this.retryStopwatch = checkNotNull(stopwatchSupplier, "stopwatchSupplier").get(); + this.node = checkNotNull(node, "node").toBuilder() + .addClientFeatures("envoy.lrs.supports_send_all_clusters").build(); logId = InternalLogId.allocate("lrs-client", targetName); logger = XdsLogger.withLogId(logId); logger.log(XdsLogLevel.INFO, "Created"); diff --git a/xds/src/test/java/io/grpc/xds/LoadReportClientTest.java b/xds/src/test/java/io/grpc/xds/LoadReportClientTest.java index 74daf52693d..574b8c42eb8 100644 --- a/xds/src/test/java/io/grpc/xds/LoadReportClientTest.java +++ b/xds/src/test/java/io/grpc/xds/LoadReportClientTest.java @@ -465,6 +465,7 @@ private static LoadStatsRequest buildInitialRequest() { .setNode( Node.newBuilder() .setId("LRS test") + .addClientFeatures("envoy.lrs.supports_send_all_clusters") .setMetadata( Struct.newBuilder() .putFields( From 7032d4ccd726243b2e4572e9ccd3be3661e90e01 Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Fri, 2 Oct 2020 14:26:25 -0700 Subject: [PATCH 79/86] xds: sync envoy proto to commit 1c27396b1f7e756ba79eed72b47f485d44da1d41 (#7480) --- xds/third_party/envoy/import.sh | 2 +- .../envoy/config/accesslog/v3/accesslog.proto | 1 + .../envoy/config/cluster/v3/cluster.proto | 27 ++++++++++++++- .../core/v3/substitution_format_string.proto | 33 ++++++++++++++----- .../config/route/v3/route_components.proto | 4 +-- .../v3/http_connection_manager.proto | 30 +++++++++-------- .../transport_sockets/tls/v3/common.proto | 9 +++-- .../transport_sockets/tls/v3/tls.proto | 29 +++++++++++++++- 8 files changed, 102 insertions(+), 33 deletions(-) diff --git a/xds/third_party/envoy/import.sh b/xds/third_party/envoy/import.sh index 593bec256c6..4e1a403eca4 100755 --- a/xds/third_party/envoy/import.sh +++ b/xds/third_party/envoy/import.sh @@ -18,7 +18,7 @@ set -e BRANCH=master # import VERSION from one of the google internal CLs -VERSION=fd28e42f31730f5ed6f13f52999692a4885dd312 +VERSION=1c27396b1f7e756ba79eed72b47f485d44da1d41 GIT_REPO="https://github.com/envoyproxy/envoy.git" GIT_BASE_DIR=envoy SOURCE_PROTO_BASE_DIR=envoy/api diff --git a/xds/third_party/envoy/src/main/proto/envoy/config/accesslog/v3/accesslog.proto b/xds/third_party/envoy/src/main/proto/envoy/config/accesslog/v3/accesslog.proto index e9d815aafce..54fa1390579 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/config/accesslog/v3/accesslog.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/config/accesslog/v3/accesslog.proto @@ -254,6 +254,7 @@ message ResponseFlagFilter { in: "UMSDR" in: "RFCF" in: "NFCF" + in: "DT" } } }]; diff --git a/xds/third_party/envoy/src/main/proto/envoy/config/cluster/v3/cluster.proto b/xds/third_party/envoy/src/main/proto/envoy/config/cluster/v3/cluster.proto index 3571ccf9abb..7747c75672c 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/config/cluster/v3/cluster.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/config/cluster/v3/cluster.proto @@ -612,7 +612,32 @@ message Cluster { // // This is limited somewhat arbitrarily to 3 because prefetching connections too aggressively can // harm latency more than the prefetching helps. - google.protobuf.DoubleValue prefetch_ratio = 1 [(validate.rules).double = {lte: 3.0 gte: 1.0}]; + google.protobuf.DoubleValue per_upstream_prefetch_ratio = 1 + [(validate.rules).double = {lte: 3.0 gte: 1.0}]; + + // Indicates how many many streams (rounded up) can be anticipated across a cluster for each + // stream, useful for low QPS services. This is currently supported for a subset of + // deterministic non-hash-based load-balancing algorithms (weighted round robin, random). + // Unlike per_upstream_prefetch_ratio this prefetches across the upstream instances in a + // cluster, doing best effort predictions of what upstream would be picked next and + // pre-establishing a connection. + // + // For example if prefetching is set to 2 for a round robin HTTP/2 cluster, on the first + // incoming stream, 2 connections will be prefetched - one to the first upstream for this + // cluster, one to the second on the assumption there will be a follow-up stream. + // + // Prefetching will be limited to one prefetch per configured upstream in the cluster. + // + // If this value is not set, or set explicitly to one, Envoy will fetch as many connections + // as needed to serve streams in flight, so during warm up and in steady state if a connection + // is closed (and per_upstream_prefetch_ratio is not set), there will be a latency hit for + // connection establishment. + // + // If both this and prefetch_ratio are set, Envoy will make sure both predicted needs are met, + // basically prefetching max(predictive-prefetch, per-upstream-prefetch), for each upstream. + // TODO(alyssawilk) per LB docs and LB overview docs when unhiding. + google.protobuf.DoubleValue predictive_prefetch_ratio = 2 + [(validate.rules).double = {lte: 3.0 gte: 1.0}]; } reserved 12, 15, 7, 11, 35; diff --git a/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/substitution_format_string.proto b/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/substitution_format_string.proto index 6c129707b2e..d3c0915c824 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/substitution_format_string.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/config/core/v3/substitution_format_string.proto @@ -23,15 +23,18 @@ message SubstitutionFormatString { // Specify a format with command operators to form a text string. // Its details is described in :ref:`format string`. // - // .. code-block:: + // For example, setting ``text_format`` like below, // - // text_format: %LOCAL_REPLY_BODY%:%RESPONSE_CODE%:path=$REQ(:path)% + // .. validated-code-block:: yaml + // :type-name: envoy.config.core.v3.SubstitutionFormatString // - // The following plain text will be created: + // text_format: "%LOCAL_REPLY_BODY%:%RESPONSE_CODE%:path=%REQ(:path)%\n" // - // .. code-block:: + // generates plain text similar to: // - // upstream connect error:204:path=/foo + // .. code-block:: text + // + // upstream connect error:503:path=/foo // string text_format = 1 [(validate.rules).string = {min_bytes: 1}]; @@ -41,11 +44,12 @@ message SubstitutionFormatString { // Nested JSON objects may be produced by some command operators (e.g. FILTER_STATE or DYNAMIC_METADATA). // See the documentation for a specific command operator for details. // - // .. code-block:: + // .. validated-code-block:: yaml + // :type-name: envoy.config.core.v3.SubstitutionFormatString // - // json_format: - // status: %RESPONSE_CODE% - // message: %LOCAL_REPLY_BODY% + // json_format: + // status: "%RESPONSE_CODE%" + // message: "%LOCAL_REPLY_BODY%" // // The following JSON object would be created: // @@ -65,4 +69,15 @@ message SubstitutionFormatString { // empty string, so that empty values are omitted entirely. // * for ``json_format`` the keys with null values are omitted in the output structure. bool omit_empty_values = 3; + + // Specify a *content_type* field. + // If this field is not set then ``text/plain`` is used for *text_format* and + // ``application/json`` is used for *json_format*. + // + // .. validated-code-block:: yaml + // :type-name: envoy.config.core.v3.SubstitutionFormatString + // + // content_type: "text/html; charset=UTF-8" + // + string content_type = 4; } diff --git a/xds/third_party/envoy/src/main/proto/envoy/config/route/v3/route_components.proto b/xds/third_party/envoy/src/main/proto/envoy/config/route/v3/route_components.proto index 0d1d7cf4c43..bd040ee31a7 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/config/route/v3/route_components.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/config/route/v3/route_components.proto @@ -778,7 +778,7 @@ message RouteAction { // `_, use that value as the // *max_stream_duration*, but limit the applied timeout to the maximum value specified here. // If set to 0, the `grpc-timeout` header is used without modification. - google.protobuf.Duration grpc_max_timeout = 2; + google.protobuf.Duration grpc_timeout_header_max = 2; // If present, Envoy will adjust the timeout provided by the `grpc-timeout` header by // subtracting the provided duration from the header. This is useful for allowing Envoy to set @@ -786,7 +786,7 @@ message RouteAction { // makes it more likely that Envoy will handle the timeout instead of having the call canceled // by the client. If, after applying the offset, the resulting timeout is zero or negative, // the stream will timeout immediately. - google.protobuf.Duration grpc_timeout_offset = 3; + google.protobuf.Duration grpc_timeout_header_offset = 3; } reserved 12, 18, 19, 16, 22, 21, 10; diff --git a/xds/third_party/envoy/src/main/proto/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto b/xds/third_party/envoy/src/main/proto/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto index 68c5c8cad2a..a715a51dbf6 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto @@ -571,27 +571,29 @@ message LocalReplyConfig { // The configuration to form response body from the :ref:`command operators ` // and to specify response content type as one of: plain/text or application/json. // - // Example one: plain/text body_format. + // Example one: "plain/text" ``body_format``. // - // .. code-block:: + // .. validated-code-block:: yaml + // :type-name: envoy.config.core.v3.SubstitutionFormatString // - // text_format: %LOCAL_REPLY_BODY%:%RESPONSE_CODE%:path=$REQ(:path)% + // text_format: "%LOCAL_REPLY_BODY%:%RESPONSE_CODE%:path=%REQ(:path)%\n" // - // The following response body in `plain/text` format will be generated for a request with + // The following response body in "plain/text" format will be generated for a request with // local reply body of "upstream connection error", response_code=503 and path=/foo. // - // .. code-block:: + // .. code-block:: text // // upstream connect error:503:path=/foo // - // Example two: application/json body_format. + // Example two: "application/json" ``body_format``. // - // .. code-block:: + // .. validated-code-block:: yaml + // :type-name: envoy.config.core.v3.SubstitutionFormatString // - // json_format: - // status: %RESPONSE_CODE% - // message: %LOCAL_REPLY_BODY% - // path: $REQ(:path)% + // json_format: + // status: "%RESPONSE_CODE%" + // message: "%LOCAL_REPLY_BODY%" + // path: "%REQ(:path)%" // // The following response body in "application/json" format would be generated for a request with // local reply body of "upstream connection error", response_code=503 and path=/foo. @@ -809,13 +811,13 @@ message HttpFilter { // sufficient. It also serves as a resource name in ExtensionConfigDS. string name = 1 [(validate.rules).string = {min_bytes: 1}]; - // Filter specific configuration which depends on the filter being instantiated. See the supported - // filters for further documentation. oneof config_type { + // Filter specific configuration which depends on the filter being instantiated. See the supported + // filters for further documentation. google.protobuf.Any typed_config = 4; // Configuration source specifier for an extension configuration discovery service. - // In case of a failure and without the default configuration, the HTTP listener responds with 500. + // In case of a failure and without the default configuration, the HTTP listener responds with code 500. // Extension configs delivered through this mechanism are not expected to require warming (see https://github.com/envoyproxy/envoy/issues/12061). config.core.v3.ExtensionConfigSource config_discovery = 5; } diff --git a/xds/third_party/envoy/src/main/proto/envoy/extensions/transport_sockets/tls/v3/common.proto b/xds/third_party/envoy/src/main/proto/envoy/extensions/transport_sockets/tls/v3/common.proto index 5eab3c1060b..3ba4e198bf1 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/extensions/transport_sockets/tls/v3/common.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/extensions/transport_sockets/tls/v3/common.proto @@ -151,7 +151,9 @@ message TlsCertificate { // TLS private key is not password encrypted. config.core.v3.DataSource password = 3 [(udpa.annotations.sensitive) = true]; - // [#not-implemented-hide:] + // The OCSP response to be stapled with this certificate during the handshake. + // The response must be DER-encoded and may only be provided via ``filename`` or + // ``inline_bytes``. The response may pertain to only one certificate. config.core.v3.DataSource ocsp_staple = 4; // [#not-implemented-hide:] @@ -205,7 +207,7 @@ message CertificateValidationContext { ACCEPT_UNTRUSTED = 1; } - reserved 4; + reserved 4, 5; reserved "verify_subject_alt_name"; @@ -315,9 +317,6 @@ message CertificateValidationContext { // `. repeated type.matcher.v3.StringMatcher match_subject_alt_names = 9; - // [#not-implemented-hide:] Must present a signed time-stamped OCSP response. - google.protobuf.BoolValue require_ocsp_staple = 5; - // [#not-implemented-hide:] Must present signed certificate time-stamp. google.protobuf.BoolValue require_signed_certificate_timestamp = 6; diff --git a/xds/third_party/envoy/src/main/proto/envoy/extensions/transport_sockets/tls/v3/tls.proto b/xds/third_party/envoy/src/main/proto/envoy/extensions/transport_sockets/tls/v3/tls.proto index f746f3d2f1c..ab716a6a42b 100644 --- a/xds/third_party/envoy/src/main/proto/envoy/extensions/transport_sockets/tls/v3/tls.proto +++ b/xds/third_party/envoy/src/main/proto/envoy/extensions/transport_sockets/tls/v3/tls.proto @@ -54,11 +54,33 @@ message UpstreamTlsContext { google.protobuf.UInt32Value max_session_keys = 4; } -// [#next-free-field: 8] +// [#next-free-field: 9] message DownstreamTlsContext { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.auth.DownstreamTlsContext"; + enum OcspStaplePolicy { + // OCSP responses are optional. If an OCSP response is absent + // or expired, the associated certificate will be used for + // connections without an OCSP staple. + LENIENT_STAPLING = 0; + + // OCSP responses are optional. If an OCSP response is absent, + // the associated certificate will be used without an + // OCSP staple. If a response is provided but is expired, + // the associated certificate will not be used for + // subsequent connections. If no suitable certificate is found, + // the connection is rejected. + STRICT_STAPLING = 1; + + // OCSP responses are required. Configuration will fail if + // a certificate is provided without an OCSP response. If a + // response expires, the associated certificate will not be + // used connections. If no suitable certificate is found, the + // connection is rejected. + MUST_STAPLE = 2; + } + // Common TLS context settings. CommonTlsContext common_tls_context = 1; @@ -96,6 +118,11 @@ message DownstreamTlsContext { lt {seconds: 4294967296} gte {} }]; + + // Config for whether to use certificates if they do not have + // an accompanying OCSP response or if the response expires at runtime. + // Defaults to LENIENT_STAPLING + OcspStaplePolicy ocsp_staple_policy = 8 [(validate.rules).enum = {defined_only: true}]; } // TLS context shared by both client and server TLS contexts. From 0f7fd289a3555b95b32e9427413cecb0a5f009cf Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Fri, 2 Oct 2020 16:50:07 -0700 Subject: [PATCH 80/86] xds: implement XdsClient APIs for watching LDS/RDS resources individually (#7470) Add XdsClient implementation of watching LDS/RDS resources, replacing the ConfigWatcher API. This makes LDS/RDS/CDS/EDS resource watchers work similarly. This change also cleans up XdsClientImpl's tests. --- xds/src/main/java/io/grpc/xds/XdsClient.java | 35 +- .../main/java/io/grpc/xds/XdsClientImpl2.java | 1702 +++++++++++++++ .../java/io/grpc/xds/XdsClientImplTest2.java | 1859 +++++++++++++++++ .../java/io/grpc/xds/XdsClientTestHelper.java | 18 + 4 files changed, 3608 insertions(+), 6 deletions(-) create mode 100644 xds/src/main/java/io/grpc/xds/XdsClientImpl2.java create mode 100644 xds/src/test/java/io/grpc/xds/XdsClientImplTest2.java diff --git a/xds/src/main/java/io/grpc/xds/XdsClient.java b/xds/src/main/java/io/grpc/xds/XdsClient.java index 01a879a2481..2021f730086 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClient.java +++ b/xds/src/main/java/io/grpc/xds/XdsClient.java @@ -16,6 +16,7 @@ package io.grpc.xds; +import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; import com.google.common.annotations.VisibleForTesting; @@ -117,7 +118,8 @@ private LdsUpdate(long httpMaxStreamDurationNano, @Nullable String rdsName, @Nullable List virtualHosts) { this.httpMaxStreamDurationNano = httpMaxStreamDurationNano; this.rdsName = rdsName; - this.virtualHosts = virtualHosts; + this.virtualHosts = virtualHosts == null + ? null : Collections.unmodifiableList(new ArrayList<>(virtualHosts)); } long getHttpMaxStreamDurationNano() { @@ -169,7 +171,7 @@ static Builder newBuilder() { return new Builder(); } - private static class Builder { + static class Builder { private long httpMaxStreamDurationNano; @Nullable private String rdsName; @@ -189,8 +191,11 @@ Builder setRdsName(String rdsName) { return this; } - Builder setVirtualHosts(List virtualHosts) { - this.virtualHosts = virtualHosts; + Builder addVirtualHost(VirtualHost virtualHost) { + if (virtualHosts == null) { + virtualHosts = new ArrayList<>(); + } + virtualHosts.add(virtualHost); return this; } @@ -206,7 +211,8 @@ static final class RdsUpdate implements ResourceUpdate { private final List virtualHosts; private RdsUpdate(List virtualHosts) { - this.virtualHosts = virtualHosts; + this.virtualHosts = Collections.unmodifiableList( + new ArrayList<>(checkNotNull(virtualHosts, "virtualHosts"))); } static RdsUpdate fromVirtualHosts(List virtualHosts) { @@ -223,6 +229,23 @@ public String toString() { .add("virtualHosts", virtualHosts) .toString(); } + + @Override + public int hashCode() { + return Objects.hash(virtualHosts); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RdsUpdate that = (RdsUpdate) o; + return Objects.equals(virtualHosts, that.virtualHosts); + } } static final class CdsUpdate implements ResourceUpdate { @@ -474,7 +497,7 @@ EdsUpdate build() { * Updates via resource discovery RPCs using LDS. Includes {@link Listener} object containing * config for security, RBAC or other server side features such as rate limit. */ - static final class ListenerUpdate { + static final class ListenerUpdate implements ResourceUpdate { // TODO(sanjaypujare): flatten structure by moving Listener class members here. private final Listener listener; diff --git a/xds/src/main/java/io/grpc/xds/XdsClientImpl2.java b/xds/src/main/java/io/grpc/xds/XdsClientImpl2.java new file mode 100644 index 00000000000..d4c7e5f58d4 --- /dev/null +++ b/xds/src/main/java/io/grpc/xds/XdsClientImpl2.java @@ -0,0 +1,1702 @@ +/* + * Copyright 2019 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.xds; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Stopwatch; +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableList; +import com.google.protobuf.Any; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.MessageOrBuilder; +import com.google.protobuf.util.Durations; +import com.google.protobuf.util.JsonFormat; +import com.google.rpc.Code; +import io.envoyproxy.envoy.config.cluster.v3.Cluster; +import io.envoyproxy.envoy.config.cluster.v3.Cluster.DiscoveryType; +import io.envoyproxy.envoy.config.cluster.v3.Cluster.EdsClusterConfig; +import io.envoyproxy.envoy.config.cluster.v3.Cluster.LbPolicy; +import io.envoyproxy.envoy.config.core.v3.Address; +import io.envoyproxy.envoy.config.core.v3.HttpProtocolOptions; +import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment; +import io.envoyproxy.envoy.config.endpoint.v3.LbEndpoint; +import io.envoyproxy.envoy.config.listener.v3.FilterChain; +import io.envoyproxy.envoy.config.listener.v3.FilterChainMatch; +import io.envoyproxy.envoy.config.listener.v3.Listener; +import io.envoyproxy.envoy.config.route.v3.RouteConfiguration; +import io.envoyproxy.envoy.config.route.v3.VirtualHost; +import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager; +import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.Rds; +import io.envoyproxy.envoy.service.discovery.v3.AggregatedDiscoveryServiceGrpc; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse; +import io.grpc.InternalLogId; +import io.grpc.Status; +import io.grpc.SynchronizationContext; +import io.grpc.SynchronizationContext.ScheduledHandle; +import io.grpc.internal.BackoffPolicy; +import io.grpc.stub.StreamObserver; +import io.grpc.xds.EnvoyProtoData.DropOverload; +import io.grpc.xds.EnvoyProtoData.Locality; +import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints; +import io.grpc.xds.EnvoyProtoData.Node; +import io.grpc.xds.EnvoyProtoData.StructOrError; +import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; +import io.grpc.xds.LoadStatsManager.LoadStatsStore; +import io.grpc.xds.XdsLogger.XdsLogLevel; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; + +final class XdsClientImpl2 extends XdsClient { + + // Longest time to wait, since the subscription to some resource, for concluding its absence. + @VisibleForTesting + static final int INITIAL_RESOURCE_FETCH_TIMEOUT_SEC = 15; + private static final String ADS_TYPE_URL_LDS_V2 = "type.googleapis.com/envoy.api.v2.Listener"; + private static final String ADS_TYPE_URL_LDS = + "type.googleapis.com/envoy.config.listener.v3.Listener"; + private static final String ADS_TYPE_URL_RDS_V2 = + "type.googleapis.com/envoy.api.v2.RouteConfiguration"; + private static final String ADS_TYPE_URL_RDS = + "type.googleapis.com/envoy.config.route.v3.RouteConfiguration"; + private static final String TYPE_URL_HTTP_CONNECTION_MANAGER_V2 = + "type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2" + + ".HttpConnectionManager"; + private static final String TYPE_URL_HTTP_CONNECTION_MANAGER = + "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3" + + ".HttpConnectionManager"; + private static final String ADS_TYPE_URL_CDS_V2 = "type.googleapis.com/envoy.api.v2.Cluster"; + private static final String ADS_TYPE_URL_CDS = + "type.googleapis.com/envoy.config.cluster.v3.Cluster"; + private static final String ADS_TYPE_URL_EDS_V2 = + "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment"; + private static final String ADS_TYPE_URL_EDS = + "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment"; + + private final MessagePrinter respPrinter = new MessagePrinter(); + private final InternalLogId logId; + private final XdsLogger logger; + private final String targetName; // TODO: delete me. + private final XdsChannel xdsChannel; + private final SynchronizationContext syncContext; + private final ScheduledExecutorService timeService; + private final BackoffPolicy.Provider backoffPolicyProvider; + private final Supplier stopwatchSupplier; + private final Stopwatch adsStreamRetryStopwatch; + // The node identifier to be included in xDS requests. Management server only requires the + // first request to carry the node identifier on a stream. It should be identical if present + // more than once. + private Node node; + + private final Map ldsResourceSubscribers = new HashMap<>(); + private final Map rdsResourceSubscribers = new HashMap<>(); + private final Map cdsResourceSubscribers = new HashMap<>(); + private final Map edsResourceSubscribers = new HashMap<>(); + + private final LoadStatsManager loadStatsManager = new LoadStatsManager(); + + // Last successfully applied version_info for each resource type. Starts with empty string. + // A version_info is used to update management server with client's most recent knowledge of + // resources. + private String ldsVersion = ""; + private String rdsVersion = ""; + private String cdsVersion = ""; + private String edsVersion = ""; + + @Nullable + private AbstractAdsStream adsStream; + @Nullable + private BackoffPolicy retryBackoffPolicy; + @Nullable + private ScheduledHandle rpcRetryTimer; + @Nullable + private LoadReportClient lrsClient; + private int loadReportCount; // number of clusters enabling load reporting + + // For server side usage. + @Nullable + private ListenerWatcher listenerWatcher; + private int listenerPort = -1; + @Nullable + private ScheduledHandle ldsRespTimer; + + XdsClientImpl2( + String targetName, + XdsChannel channel, + Node node, + SynchronizationContext syncContext, + ScheduledExecutorService timeService, + BackoffPolicy.Provider backoffPolicyProvider, + Supplier stopwatchSupplier) { + this.targetName = checkNotNull(targetName, "targetName"); + this.xdsChannel = checkNotNull(channel, "channel"); + this.node = checkNotNull(node, "node"); + this.syncContext = checkNotNull(syncContext, "syncContext"); + this.timeService = checkNotNull(timeService, "timeService"); + this.backoffPolicyProvider = checkNotNull(backoffPolicyProvider, "backoffPolicyProvider"); + this.stopwatchSupplier = checkNotNull(stopwatchSupplier, "stopwatch"); + adsStreamRetryStopwatch = stopwatchSupplier.get(); + logId = InternalLogId.allocate("xds-client", null); + logger = XdsLogger.withLogId(logId); + logger.log(XdsLogLevel.INFO, "Created"); + } + + @Override + void shutdown() { + logger.log(XdsLogLevel.INFO, "Shutting down"); + xdsChannel.getManagedChannel().shutdown(); + if (adsStream != null) { + adsStream.close(Status.CANCELLED.withDescription("shutdown").asException()); + } + cleanUpResourceTimers(); + if (lrsClient != null) { + lrsClient.stopLoadReporting(); + lrsClient = null; + } + if (rpcRetryTimer != null) { + rpcRetryTimer.cancel(); + } + } + + private void cleanUpResourceTimers() { + if (ldsRespTimer != null) { + ldsRespTimer.cancel(); + ldsRespTimer = null; + } + for (ResourceSubscriber subscriber : ldsResourceSubscribers.values()) { + subscriber.stopTimer(); + } + for (ResourceSubscriber subscriber : rdsResourceSubscribers.values()) { + subscriber.stopTimer(); + } + for (ResourceSubscriber subscriber : cdsResourceSubscribers.values()) { + subscriber.stopTimer(); + } + for (ResourceSubscriber subscriber : edsResourceSubscribers.values()) { + subscriber.stopTimer(); + } + } + + @Override + void watchLdsResource(String resourceName, LdsResourceWatcher watcher) { + ResourceSubscriber subscriber = ldsResourceSubscribers.get(resourceName); + if (subscriber == null) { + logger.log(XdsLogLevel.INFO, "Subscribe CDS resource {0}", resourceName); + subscriber = new ResourceSubscriber(ResourceType.LDS, resourceName); + ldsResourceSubscribers.put(resourceName, subscriber); + adjustResourceSubscription(ResourceType.LDS, ldsResourceSubscribers.keySet()); + } + subscriber.addWatcher(watcher); + } + + @Override + void cancelLdsResourceWatch(String resourceName, LdsResourceWatcher watcher) { + ResourceSubscriber subscriber = ldsResourceSubscribers.get(resourceName); + subscriber.removeWatcher(watcher); + if (!subscriber.isWatched()) { + subscriber.stopTimer(); + logger.log(XdsLogLevel.INFO, "Unsubscribe LDS resource {0}", resourceName); + ldsResourceSubscribers.remove(resourceName); + adjustResourceSubscription(ResourceType.LDS, ldsResourceSubscribers.keySet()); + } + } + + @Override + void watchRdsResource(String resourceName, RdsResourceWatcher watcher) { + ResourceSubscriber subscriber = rdsResourceSubscribers.get(resourceName); + if (subscriber == null) { + logger.log(XdsLogLevel.INFO, "Subscribe RDS resource {0}", resourceName); + subscriber = new ResourceSubscriber(ResourceType.RDS, resourceName); + rdsResourceSubscribers.put(resourceName, subscriber); + adjustResourceSubscription(ResourceType.RDS, rdsResourceSubscribers.keySet()); + } + subscriber.addWatcher(watcher); + } + + @Override + void cancelRdsResourceWatch(String resourceName, RdsResourceWatcher watcher) { + ResourceSubscriber subscriber = rdsResourceSubscribers.get(resourceName); + subscriber.removeWatcher(watcher); + if (!subscriber.isWatched()) { + subscriber.stopTimer(); + logger.log(XdsLogLevel.INFO, "Unsubscribe RDS resource {0}", resourceName); + rdsResourceSubscribers.remove(resourceName); + adjustResourceSubscription(ResourceType.RDS, rdsResourceSubscribers.keySet()); + } + } + + @Override + void watchCdsResource(String resourceName, CdsResourceWatcher watcher) { + ResourceSubscriber subscriber = cdsResourceSubscribers.get(resourceName); + if (subscriber == null) { + logger.log(XdsLogLevel.INFO, "Subscribe CDS resource {0}", resourceName); + subscriber = new ResourceSubscriber(ResourceType.CDS, resourceName); + cdsResourceSubscribers.put(resourceName, subscriber); + adjustResourceSubscription(ResourceType.CDS, cdsResourceSubscribers.keySet()); + } + subscriber.addWatcher(watcher); + } + + @Override + void cancelCdsResourceWatch(String resourceName, CdsResourceWatcher watcher) { + ResourceSubscriber subscriber = cdsResourceSubscribers.get(resourceName); + subscriber.removeWatcher(watcher); + if (!subscriber.isWatched()) { + subscriber.stopTimer(); + logger.log(XdsLogLevel.INFO, "Unsubscribe CDS resource {0}", resourceName); + cdsResourceSubscribers.remove(resourceName); + adjustResourceSubscription(ResourceType.CDS, cdsResourceSubscribers.keySet()); + } + } + + @Override + void watchEdsResource(String resourceName, EdsResourceWatcher watcher) { + ResourceSubscriber subscriber = edsResourceSubscribers.get(resourceName); + if (subscriber == null) { + logger.log(XdsLogLevel.INFO, "Subscribe EDS resource {0}", resourceName); + subscriber = new ResourceSubscriber(ResourceType.EDS, resourceName); + edsResourceSubscribers.put(resourceName, subscriber); + adjustResourceSubscription(ResourceType.EDS, edsResourceSubscribers.keySet()); + } + subscriber.addWatcher(watcher); + } + + @Override + void cancelEdsResourceWatch(String resourceName, EdsResourceWatcher watcher) { + ResourceSubscriber subscriber = edsResourceSubscribers.get(resourceName); + subscriber.removeWatcher(watcher); + if (!subscriber.isWatched()) { + subscriber.stopTimer(); + logger.log(XdsLogLevel.INFO, "Unsubscribe EDS resource {0}", resourceName); + edsResourceSubscribers.remove(resourceName); + adjustResourceSubscription(ResourceType.EDS, edsResourceSubscribers.keySet()); + } + } + + @Override + void watchListenerData(int port, ListenerWatcher watcher) { + checkState(listenerWatcher == null, "ListenerWatcher already registered"); + listenerWatcher = checkNotNull(watcher, "watcher"); + checkArgument(port > 0, "port needs to be > 0"); + this.listenerPort = port; + logger.log(XdsLogLevel.INFO, "Started watching listener for port {0}", port); + if (rpcRetryTimer != null && rpcRetryTimer.isPending()) { + // Currently in retry backoff. + return; + } + if (adsStream == null) { + startRpcStream(); + } + updateNodeMetadataForListenerRequest(port); + adsStream.sendXdsRequest(ResourceType.LDS, ImmutableList.of()); + ldsRespTimer = + syncContext + .schedule( + new ListenerResourceFetchTimeoutTask(":" + port), + INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS, timeService); + } + + /** In case of Listener watcher metadata to be updated to include port. */ + private void updateNodeMetadataForListenerRequest(int port) { + Map newMetadata = new HashMap<>(); + if (node.getMetadata() != null) { + newMetadata.putAll(node.getMetadata()); + } + newMetadata.put("TRAFFICDIRECTOR_PROXYLESS", "1"); + // TODO(sanjaypujare): eliminate usage of listening_addresses. + EnvoyProtoData.Address listeningAddress = + new EnvoyProtoData.Address("0.0.0.0", port); + node = + node.toBuilder().setMetadata(newMetadata).addListeningAddresses(listeningAddress).build(); + } + + @Override + void reportClientStats() { + if (lrsClient == null) { + logger.log(XdsLogLevel.INFO, "Turning on load reporting"); + lrsClient = + new LoadReportClient( + targetName, + loadStatsManager, + xdsChannel, + node, + syncContext, + timeService, + backoffPolicyProvider, + stopwatchSupplier); + } + if (loadReportCount == 0) { + lrsClient.startLoadReporting(); + } + loadReportCount++; + } + + @Override + void cancelClientStatsReport() { + checkState(loadReportCount > 0, "load reporting was never started"); + loadReportCount--; + if (loadReportCount == 0) { + logger.log(XdsLogLevel.INFO, "Turning off load reporting"); + lrsClient.stopLoadReporting(); + lrsClient = null; + } + } + + @Override + LoadStatsStore addClientStats(String clusterName, @Nullable String clusterServiceName) { + return loadStatsManager.addLoadStats(clusterName, clusterServiceName); + } + + @Override + void removeClientStats(String clusterName, @Nullable String clusterServiceName) { + loadStatsManager.removeLoadStats(clusterName, clusterServiceName); + } + + @Override + public String toString() { + return logId.toString(); + } + + /** + * Establishes the RPC connection by creating a new RPC stream on the given channel for + * xDS protocol communication. + */ + private void startRpcStream() { + checkState(adsStream == null, "Previous adsStream has not been cleared yet"); + if (xdsChannel.isUseProtocolV3()) { + adsStream = new AdsStream(); + } else { + adsStream = new AdsStreamV2(); + } + adsStream.start(); + logger.log(XdsLogLevel.INFO, "ADS stream started"); + adsStreamRetryStopwatch.reset().start(); + } + + private void handleLdsResponse(DiscoveryResponseData ldsResponse) { + if (listenerWatcher != null) { + handleLdsResponseForServer(ldsResponse); + } else { + handleLdsResponseForClient(ldsResponse); + } + } + + private void handleLdsResponseForClient(DiscoveryResponseData ldsResponse) { + // Unpack Listener messages. + List listeners = new ArrayList<>(ldsResponse.getResourcesList().size()); + List listenerNames = new ArrayList<>(ldsResponse.getResourcesList().size()); + try { + for (com.google.protobuf.Any res : ldsResponse.getResourcesList()) { + if (res.getTypeUrl().equals(ADS_TYPE_URL_LDS_V2)) { + res = res.toBuilder().setTypeUrl(ADS_TYPE_URL_LDS).build(); + } + Listener listener = res.unpack(Listener.class); + listeners.add(listener); + listenerNames.add(listener.getName()); + } + } catch (InvalidProtocolBufferException e) { + logger.log(XdsLogLevel.WARNING, "Failed to unpack Listeners in LDS response {0}", e); + adsStream.sendNackRequest( + ResourceType.LDS, ldsResourceSubscribers.keySet(), + ldsResponse.getVersionInfo(), "Malformed LDS response: " + e); + return; + } + logger.log(XdsLogLevel.INFO, "Received LDS response for resources: {0}", listenerNames); + + // Unpack HttpConnectionManager messages. + Map httpConnectionManagers = new HashMap<>(listeners.size()); + try { + for (Listener listener : listeners) { + Any apiListener = listener.getApiListener().getApiListener(); + if (apiListener.getTypeUrl().equals(TYPE_URL_HTTP_CONNECTION_MANAGER_V2)) { + apiListener = + apiListener.toBuilder().setTypeUrl(TYPE_URL_HTTP_CONNECTION_MANAGER).build(); + } + HttpConnectionManager hcm = apiListener.unpack(HttpConnectionManager.class); + httpConnectionManagers.put(listener.getName(), hcm); + } + } catch (InvalidProtocolBufferException e) { + logger.log( + XdsLogLevel.WARNING, + "Failed to unpack HttpConnectionManagers in Listeners of LDS response {0}", e); + adsStream.sendNackRequest( + ResourceType.LDS, ldsResourceSubscribers.keySet(), + ldsResponse.getVersionInfo(), "Malformed LDS response: " + e); + return; + } + + Map ldsUpdates = new HashMap<>(); + Set rdsNames = new HashSet<>(); + String errorMessage = null; + for (Map.Entry entry : httpConnectionManagers.entrySet()) { + String listenerName = entry.getKey(); + HttpConnectionManager hcm = entry.getValue(); + LdsUpdate.Builder updateBuilder = LdsUpdate.newBuilder(); + if (hcm.hasRouteConfig()) { + for (VirtualHost virtualHostProto : hcm.getRouteConfig().getVirtualHostsList()) { + StructOrError virtualHost = + EnvoyProtoData.VirtualHost.fromEnvoyProtoVirtualHost(virtualHostProto); + if (virtualHost.getErrorDetail() != null) { + errorMessage = "Listener " + listenerName + " contains invalid virtual host: " + + virtualHost.getErrorDetail(); + break; + } else { + updateBuilder.addVirtualHost(virtualHost.getStruct()); + } + } + } else if (hcm.hasRds()) { + Rds rds = hcm.getRds(); + if (!rds.getConfigSource().hasAds()) { + errorMessage = "Listener " + listenerName + " with RDS config_source not set to ADS"; + } else { + updateBuilder.setRdsName(rds.getRouteConfigName()); + rdsNames.add(rds.getRouteConfigName()); + } + } else { + errorMessage = "Listener " + listenerName + " without inline RouteConfiguration or RDS"; + } + if (errorMessage != null) { + break; + } + if (hcm.hasCommonHttpProtocolOptions()) { + HttpProtocolOptions options = hcm.getCommonHttpProtocolOptions(); + if (options.hasMaxStreamDuration()) { + updateBuilder.setHttpMaxStreamDurationNano( + Durations.toNanos(options.getMaxStreamDuration())); + } + } + ldsUpdates.put(listenerName, updateBuilder.build()); + } + if (errorMessage != null) { + adsStream.sendNackRequest( + ResourceType.LDS, ldsResourceSubscribers.keySet(), + ldsResponse.getVersionInfo(), errorMessage); + return; + } + adsStream.sendAckRequest( + ResourceType.LDS, ldsResourceSubscribers.keySet(), ldsResponse.getVersionInfo()); + + for (String resource : ldsResourceSubscribers.keySet()) { + ResourceSubscriber subscriber = ldsResourceSubscribers.get(resource); + if (ldsUpdates.containsKey(resource)) { + subscriber.onData(ldsUpdates.get(resource)); + } else { + subscriber.onAbsent(); + } + } + for (String resource : rdsResourceSubscribers.keySet()) { + if (!rdsNames.contains(resource)) { + ResourceSubscriber subscriber = rdsResourceSubscribers.get(resource); + subscriber.onAbsent(); + } + } + } + + private void handleRdsResponse(DiscoveryResponseData rdsResponse) { + // Unpack RouteConfiguration messages. + Map routeConfigs = + new HashMap<>(rdsResponse.getResourcesList().size()); + try { + for (com.google.protobuf.Any res : rdsResponse.getResourcesList()) { + if (res.getTypeUrl().equals(ADS_TYPE_URL_RDS_V2)) { + res = res.toBuilder().setTypeUrl(ADS_TYPE_URL_RDS).build(); + } + RouteConfiguration rc = res.unpack(RouteConfiguration.class); + routeConfigs.put(rc.getName(), rc); + } + } catch (InvalidProtocolBufferException e) { + logger.log( + XdsLogLevel.WARNING, "Failed to unpack RouteConfiguration in RDS response {0}", e); + adsStream.sendNackRequest( + ResourceType.RDS, rdsResourceSubscribers.keySet(), + rdsResponse.getVersionInfo(), "Malformed RDS response: " + e); + return; + } + logger.log( + XdsLogLevel.INFO, "Received RDS response for resources: {0}", routeConfigs.keySet()); + + Map rdsUpdates = new HashMap<>(); + String errorMessage = null; + for (Map.Entry entry : routeConfigs.entrySet()) { + String routeConfigName = entry.getKey(); + RouteConfiguration routeConfig = entry.getValue(); + List virtualHosts = + new ArrayList<>(routeConfig.getVirtualHostsCount()); + for (VirtualHost virtualHostProto : routeConfig.getVirtualHostsList()) { + StructOrError virtualHost = + EnvoyProtoData.VirtualHost.fromEnvoyProtoVirtualHost(virtualHostProto); + if (virtualHost.getErrorDetail() != null) { + errorMessage = "RouteConfiguration " + routeConfigName + + " contains invalid virtual host: " + virtualHost.getErrorDetail(); + break; + } else { + virtualHosts.add(virtualHost.getStruct()); + } + } + if (errorMessage != null) { + break; + } + rdsUpdates.put(routeConfigName, RdsUpdate.fromVirtualHosts(virtualHosts)); + } + if (errorMessage != null) { + adsStream.sendNackRequest(ResourceType.RDS, rdsResourceSubscribers.keySet(), + rdsResponse.getVersionInfo(), errorMessage); + return; + } + adsStream.sendAckRequest(ResourceType.RDS, rdsResourceSubscribers.keySet(), + rdsResponse.getVersionInfo()); + + for (String resource : rdsResourceSubscribers.keySet()) { + if (rdsUpdates.containsKey(resource)) { + ResourceSubscriber subscriber = rdsResourceSubscribers.get(resource); + subscriber.onData(rdsUpdates.get(resource)); + } + } + } + + private void handleLdsResponseForServer(DiscoveryResponseData ldsResponse) { + // Unpack Listener messages. + Listener requestedListener = null; + logger.log(XdsLogLevel.DEBUG, "Listener count: {0}", ldsResponse.getResourcesList().size()); + try { + for (com.google.protobuf.Any res : ldsResponse.getResourcesList()) { + if (res.getTypeUrl().equals(ADS_TYPE_URL_LDS_V2)) { + res = res.toBuilder().setTypeUrl(ADS_TYPE_URL_LDS).build(); + } + Listener listener = res.unpack(Listener.class); + logger.log(XdsLogLevel.DEBUG, "Found listener {0}", listener.toString()); + if (isRequestedListener(listener)) { + requestedListener = listener; + logger.log(XdsLogLevel.DEBUG, "Requested listener found: {0}", listener.getName()); + } + } + } catch (InvalidProtocolBufferException e) { + logger.log(XdsLogLevel.WARNING, "Failed to unpack Listeners in LDS response {0}", e); + adsStream.sendNackRequest( + ResourceType.LDS, ImmutableList.of(), + ldsResponse.getVersionInfo(), "Malformed LDS response: " + e); + return; + } + ListenerUpdate listenerUpdate = null; + if (requestedListener != null) { + if (ldsRespTimer != null) { + ldsRespTimer.cancel(); + ldsRespTimer = null; + } + try { + listenerUpdate = ListenerUpdate.newBuilder() + .setListener(EnvoyServerProtoData.Listener.fromEnvoyProtoListener(requestedListener)) + .build(); + } catch (InvalidProtocolBufferException e) { + logger.log(XdsLogLevel.WARNING, "Failed to unpack Listener in LDS response {0}", e); + adsStream.sendNackRequest( + ResourceType.LDS, ImmutableList.of(), + ldsResponse.getVersionInfo(), "Malformed LDS response: " + e); + return; + } + } else { + if (ldsRespTimer == null) { + listenerWatcher.onResourceDoesNotExist(":" + listenerPort); + } + } + adsStream.sendAckRequest(ResourceType.LDS, ImmutableList.of(), + ldsResponse.getVersionInfo()); + if (listenerUpdate != null) { + listenerWatcher.onListenerChanged(listenerUpdate); + } + } + + private boolean isRequestedListener(Listener listener) { + // TODO(sanjaypujare): check listener.getName() once we know what xDS server returns + return isAddressMatching(listener.getAddress()) + && hasMatchingFilter(listener.getFilterChainsList()); + } + + private boolean isAddressMatching(Address address) { + // TODO(sanjaypujare): check IP address once we know xDS server will include it + return address.hasSocketAddress() + && (address.getSocketAddress().getPortValue() == listenerPort); + } + + private boolean hasMatchingFilter(List filterChainsList) { + // TODO(sanjaypujare): if myIp to be checked against filterChainMatch.getPrefixRangesList() + for (FilterChain filterChain : filterChainsList) { + FilterChainMatch filterChainMatch = filterChain.getFilterChainMatch(); + + if (listenerPort == filterChainMatch.getDestinationPort().getValue()) { + return true; + } + } + return false; + } + + /** + * Handles CDS response, which contains a list of Cluster messages with information for a logical + * cluster. The response is NACKed if messages for requested resources contain invalid + * information for gRPC's usage. Otherwise, an ACK request is sent to management server. + * Response data for requested clusters is cached locally, in case of new cluster watchers + * interested in the same clusters are added later. + */ + private void handleCdsResponse(DiscoveryResponseData cdsResponse) { + adsStream.cdsRespNonce = cdsResponse.getNonce(); + + // Unpack Cluster messages. + List clusters = new ArrayList<>(cdsResponse.getResourcesList().size()); + List clusterNames = new ArrayList<>(cdsResponse.getResourcesList().size()); + try { + for (com.google.protobuf.Any res : cdsResponse.getResourcesList()) { + if (res.getTypeUrl().equals(ADS_TYPE_URL_CDS_V2)) { + res = res.toBuilder().setTypeUrl(ADS_TYPE_URL_CDS).build(); + } + Cluster cluster = res.unpack(Cluster.class); + clusters.add(cluster); + clusterNames.add(cluster.getName()); + } + } catch (InvalidProtocolBufferException e) { + logger.log(XdsLogLevel.WARNING, "Failed to unpack Clusters in CDS response {0}", e); + adsStream.sendNackRequest( + ResourceType.CDS, cdsResourceSubscribers.keySet(), + cdsResponse.getVersionInfo(), "Malformed CDS response: " + e); + return; + } + logger.log(XdsLogLevel.INFO, "Received CDS response for resources: {0}", clusterNames); + + String errorMessage = null; + // Cluster information update for requested clusters received in this CDS response. + Map cdsUpdates = new HashMap<>(); + // CDS responses represents the state of the world, EDS services not referenced by + // Clusters are those no longer exist. + Set edsServices = new HashSet<>(); + for (Cluster cluster : clusters) { + String clusterName = cluster.getName(); + // Skip information for clusters not requested. + // Management server is required to always send newly requested resources, even if they + // may have been sent previously (proactively). Thus, client does not need to cache + // unrequested resources. + if (!cdsResourceSubscribers.containsKey(clusterName)) { + continue; + } + CdsUpdate.Builder updateBuilder = CdsUpdate.newBuilder(); + updateBuilder.setClusterName(clusterName); + // The type field must be set to EDS. + if (!cluster.getType().equals(DiscoveryType.EDS)) { + errorMessage = "Cluster " + clusterName + " : only EDS discovery type is supported " + + "in gRPC."; + break; + } + // In the eds_cluster_config field, the eds_config field must be set to indicate to + // use EDS (must be set to use ADS). + EdsClusterConfig edsClusterConfig = cluster.getEdsClusterConfig(); + if (!edsClusterConfig.getEdsConfig().hasAds()) { + errorMessage = "Cluster " + clusterName + " : field eds_cluster_config must be set to " + + "indicate to use EDS over ADS."; + break; + } + // If the service_name field is set, that value will be used for the EDS request. + if (!edsClusterConfig.getServiceName().isEmpty()) { + updateBuilder.setEdsServiceName(edsClusterConfig.getServiceName()); + edsServices.add(edsClusterConfig.getServiceName()); + } else { + edsServices.add(clusterName); + } + // The lb_policy field must be set to ROUND_ROBIN. + if (!cluster.getLbPolicy().equals(LbPolicy.ROUND_ROBIN)) { + errorMessage = "Cluster " + clusterName + " : only round robin load balancing policy is " + + "supported in gRPC."; + break; + } + updateBuilder.setLbPolicy("round_robin"); + // If the lrs_server field is set, it must have its self field set, in which case the + // client should use LRS for load reporting. Otherwise (the lrs_server field is not set), + // LRS load reporting will be disabled. + if (cluster.hasLrsServer()) { + if (!cluster.getLrsServer().hasSelf()) { + errorMessage = "Cluster " + clusterName + " : only support enabling LRS for the same " + + "management server."; + break; + } + updateBuilder.setLrsServerName(""); + } + try { + UpstreamTlsContext upstreamTlsContext = getTlsContextFromCluster(cluster); + if (upstreamTlsContext != null && upstreamTlsContext.getCommonTlsContext() != null) { + updateBuilder.setUpstreamTlsContext(upstreamTlsContext); + } + } catch (InvalidProtocolBufferException e) { + errorMessage = "Cluster " + clusterName + " : " + e.getMessage(); + break; + } + cdsUpdates.put(clusterName, updateBuilder.build()); + } + if (errorMessage != null) { + adsStream.sendNackRequest( + ResourceType.CDS, + cdsResourceSubscribers.keySet(), + cdsResponse.getVersionInfo(), + errorMessage); + return; + } + adsStream.sendAckRequest(ResourceType.CDS, cdsResourceSubscribers.keySet(), + cdsResponse.getVersionInfo()); + + for (String resource : cdsResourceSubscribers.keySet()) { + ResourceSubscriber subscriber = cdsResourceSubscribers.get(resource); + if (cdsUpdates.containsKey(resource)) { + subscriber.onData(cdsUpdates.get(resource)); + } else { + subscriber.onAbsent(); + } + } + for (String resource : edsResourceSubscribers.keySet()) { + ResourceSubscriber subscriber = edsResourceSubscribers.get(resource); + if (!edsServices.contains(resource)) { + subscriber.onAbsent(); + } + } + } + + @Nullable + private static UpstreamTlsContext getTlsContextFromCluster(Cluster cluster) + throws InvalidProtocolBufferException { + if (cluster.hasTransportSocket() && "tls".equals(cluster.getTransportSocket().getName())) { + Any any = cluster.getTransportSocket().getTypedConfig(); + return UpstreamTlsContext.fromEnvoyProtoUpstreamTlsContext( + io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext.parseFrom( + any.getValue())); + } + return null; + } + + /** + * Handles EDS response, which contains a list of ClusterLoadAssignment messages with + * endpoint load balancing information for each cluster. The response is NACKed if messages + * for requested resources contain invalid information for gRPC's usage. Otherwise, + * an ACK request is sent to management server. Response data for requested clusters is + * cached locally, in case of new endpoint watchers interested in the same clusters + * are added later. + */ + private void handleEdsResponse(DiscoveryResponseData edsResponse) { + // Unpack ClusterLoadAssignment messages. + List clusterLoadAssignments = + new ArrayList<>(edsResponse.getResourcesList().size()); + List claNames = new ArrayList<>(edsResponse.getResourcesList().size()); + try { + for (com.google.protobuf.Any res : edsResponse.getResourcesList()) { + if (res.getTypeUrl().equals(ADS_TYPE_URL_EDS_V2)) { + res = res.toBuilder().setTypeUrl(ADS_TYPE_URL_EDS).build(); + } + ClusterLoadAssignment assignment = res.unpack(ClusterLoadAssignment.class); + clusterLoadAssignments.add(assignment); + claNames.add(assignment.getClusterName()); + } + } catch (InvalidProtocolBufferException e) { + logger.log( + XdsLogLevel.WARNING, "Failed to unpack ClusterLoadAssignments in EDS response {0}", e); + adsStream.sendNackRequest( + ResourceType.EDS, edsResourceSubscribers.keySet(), + edsResponse.getVersionInfo(), "Malformed EDS response: " + e); + return; + } + logger.log(XdsLogLevel.INFO, "Received EDS response for resources: {0}", claNames); + + String errorMessage = null; + // Endpoint information updates for requested clusters received in this EDS response. + Map edsUpdates = new HashMap<>(); + // Walk through each ClusterLoadAssignment message. If any of them for requested clusters + // contain invalid information for gRPC's load balancing usage, the whole response is rejected. + for (ClusterLoadAssignment assignment : clusterLoadAssignments) { + String clusterName = assignment.getClusterName(); + // Skip information for clusters not requested. + // Management server is required to always send newly requested resources, even if they + // may have been sent previously (proactively). Thus, client does not need to cache + // unrequested resources. + if (!edsResourceSubscribers.containsKey(clusterName)) { + continue; + } + EdsUpdate.Builder updateBuilder = EdsUpdate.newBuilder(); + updateBuilder.setClusterName(clusterName); + Set priorities = new HashSet<>(); + int maxPriority = -1; + for (io.envoyproxy.envoy.config.endpoint.v3.LocalityLbEndpoints localityLbEndpoints + : assignment.getEndpointsList()) { + // Filter out localities without or with 0 weight. + if (!localityLbEndpoints.hasLoadBalancingWeight() + || localityLbEndpoints.getLoadBalancingWeight().getValue() < 1) { + continue; + } + int localityPriority = localityLbEndpoints.getPriority(); + if (localityPriority < 0) { + errorMessage = + "ClusterLoadAssignment " + clusterName + " : locality with negative priority."; + break; + } + maxPriority = Math.max(maxPriority, localityPriority); + priorities.add(localityPriority); + // The endpoint field of each lb_endpoints must be set. + // Inside of it: the address field must be set. + for (LbEndpoint lbEndpoint : localityLbEndpoints.getLbEndpointsList()) { + if (!lbEndpoint.getEndpoint().hasAddress()) { + errorMessage = "ClusterLoadAssignment " + clusterName + " : endpoint with no address."; + break; + } + } + if (errorMessage != null) { + break; + } + // Note endpoints with health status other than UNHEALTHY and UNKNOWN are still + // handed over to watching parties. It is watching parties' responsibility to + // filter out unhealthy endpoints. See EnvoyProtoData.LbEndpoint#isHealthy(). + updateBuilder.addLocalityLbEndpoints( + Locality.fromEnvoyProtoLocality(localityLbEndpoints.getLocality()), + LocalityLbEndpoints.fromEnvoyProtoLocalityLbEndpoints(localityLbEndpoints)); + } + if (errorMessage != null) { + break; + } + if (priorities.size() != maxPriority + 1) { + errorMessage = "ClusterLoadAssignment " + clusterName + " : sparse priorities."; + break; + } + for (ClusterLoadAssignment.Policy.DropOverload dropOverload + : assignment.getPolicy().getDropOverloadsList()) { + updateBuilder.addDropPolicy(DropOverload.fromEnvoyProtoDropOverload(dropOverload)); + } + EdsUpdate update = updateBuilder.build(); + edsUpdates.put(clusterName, update); + } + if (errorMessage != null) { + adsStream.sendNackRequest( + ResourceType.EDS, + edsResourceSubscribers.keySet(), + edsResponse.getVersionInfo(), + errorMessage); + return; + } + adsStream.sendAckRequest(ResourceType.EDS, edsResourceSubscribers.keySet(), + edsResponse.getVersionInfo()); + + for (String resource : edsResourceSubscribers.keySet()) { + ResourceSubscriber subscriber = edsResourceSubscribers.get(resource); + if (edsUpdates.containsKey(resource)) { + subscriber.onData(edsUpdates.get(resource)); + } + } + } + + private void adjustResourceSubscription(ResourceType type, Collection resources) { + if (rpcRetryTimer != null && rpcRetryTimer.isPending()) { + // Currently in retry backoff. + return; + } + if (adsStream == null) { + startRpcStream(); + } + adsStream.sendXdsRequest(type, resources); + } + + @VisibleForTesting + final class RpcRetryTask implements Runnable { + @Override + public void run() { + startRpcStream(); + if (listenerWatcher != null) { + adsStream.sendXdsRequest(ResourceType.LDS, ImmutableList.of()); + ldsRespTimer = + syncContext + .schedule( + new ListenerResourceFetchTimeoutTask(":" + listenerPort), + INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS, timeService); + } + if (!ldsResourceSubscribers.isEmpty()) { + adsStream.sendXdsRequest(ResourceType.LDS, ldsResourceSubscribers.keySet()); + for (ResourceSubscriber subscriber : ldsResourceSubscribers.values()) { + subscriber.restartTimer(); + } + } + if (!rdsResourceSubscribers.isEmpty()) { + adsStream.sendXdsRequest(ResourceType.RDS, rdsResourceSubscribers.keySet()); + for (ResourceSubscriber subscriber : rdsResourceSubscribers.values()) { + subscriber.restartTimer(); + } + } + if (!cdsResourceSubscribers.isEmpty()) { + adsStream.sendXdsRequest(ResourceType.CDS, cdsResourceSubscribers.keySet()); + for (ResourceSubscriber subscriber : cdsResourceSubscribers.values()) { + subscriber.restartTimer(); + } + } + if (!edsResourceSubscribers.isEmpty()) { + adsStream.sendXdsRequest(ResourceType.EDS, edsResourceSubscribers.keySet()); + for (ResourceSubscriber subscriber : edsResourceSubscribers.values()) { + subscriber.restartTimer(); + } + } + } + } + + @VisibleForTesting + enum ResourceType { + UNKNOWN, LDS, RDS, CDS, EDS; + + @VisibleForTesting + String typeUrl() { + switch (this) { + case LDS: + return ADS_TYPE_URL_LDS; + case RDS: + return ADS_TYPE_URL_RDS; + case CDS: + return ADS_TYPE_URL_CDS; + case EDS: + return ADS_TYPE_URL_EDS; + case UNKNOWN: + default: + throw new AssertionError("Unknown or missing case in enum switch: " + this); + } + } + + private String typeUrlV2() { + switch (this) { + case LDS: + return ADS_TYPE_URL_LDS_V2; + case RDS: + return ADS_TYPE_URL_RDS_V2; + case CDS: + return ADS_TYPE_URL_CDS_V2; + case EDS: + return ADS_TYPE_URL_EDS_V2; + case UNKNOWN: + default: + throw new AssertionError("Unknown or missing case in enum switch: " + this); + } + } + + private static ResourceType fromTypeUrl(String typeUrl) { + switch (typeUrl) { + case ADS_TYPE_URL_LDS: + // fall trough + case ADS_TYPE_URL_LDS_V2: + return LDS; + case ADS_TYPE_URL_RDS: + // fall through + case ADS_TYPE_URL_RDS_V2: + return RDS; + case ADS_TYPE_URL_CDS: + // fall through + case ADS_TYPE_URL_CDS_V2: + return CDS; + case ADS_TYPE_URL_EDS: + // fall through + case ADS_TYPE_URL_EDS_V2: + return EDS; + default: + return UNKNOWN; + } + } + } + + /** + * Tracks a single subscribed resource. + */ + private final class ResourceSubscriber { + private final ResourceType type; + private final String resource; + private final Set watchers = new HashSet<>(); + private ResourceUpdate data; + private boolean absent; + private ScheduledHandle respTimer; + + ResourceSubscriber(ResourceType type, String resource) { + this.type = type; + this.resource = resource; + if (rpcRetryTimer != null && rpcRetryTimer.isPending()) { + return; + } + restartTimer(); + } + + void addWatcher(ResourceWatcher watcher) { + checkArgument(!watchers.contains(watcher), "watcher %s already registered", watcher); + watchers.add(watcher); + if (data != null) { + notifyWatcher(watcher, data); + } else if (absent) { + watcher.onResourceDoesNotExist(resource); + } + } + + void removeWatcher(ResourceWatcher watcher) { + checkArgument(watchers.contains(watcher), "watcher %s not registered", watcher); + watchers.remove(watcher); + } + + void restartTimer() { + class ResourceNotFound implements Runnable { + @Override + public void run() { + logger.log(XdsLogLevel.INFO, "{0} resource {1} initial fetch timeout", type, resource); + respTimer = null; + onAbsent(); + } + + @Override + public String toString() { + return type + this.getClass().getSimpleName(); + } + } + + respTimer = syncContext.schedule( + new ResourceNotFound(), INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS, + timeService); + } + + void stopTimer() { + if (respTimer != null && respTimer.isPending()) { + respTimer.cancel(); + respTimer = null; + } + } + + boolean isWatched() { + return !watchers.isEmpty(); + } + + void onData(ResourceUpdate data) { + if (respTimer != null && respTimer.isPending()) { + respTimer.cancel(); + respTimer = null; + } + ResourceUpdate oldData = this.data; + this.data = data; + absent = false; + if (!Objects.equals(oldData, data)) { + for (ResourceWatcher watcher : watchers) { + notifyWatcher(watcher, data); + } + } + } + + void onAbsent() { + if (respTimer != null && respTimer.isPending()) { // too early to conclude absence + return; + } + logger.log(XdsLogLevel.INFO, "Conclude {0} resource {1} not exist", type, resource); + if (!absent) { + data = null; + absent = true; + for (ResourceWatcher watcher : watchers) { + watcher.onResourceDoesNotExist(resource); + } + } + } + + void onError(Status error) { + if (respTimer != null && respTimer.isPending()) { + respTimer.cancel(); + respTimer = null; + } + for (ResourceWatcher watcher : watchers) { + watcher.onError(error); + } + } + + private void notifyWatcher(ResourceWatcher watcher, ResourceUpdate update) { + switch (type) { + case LDS: + ((LdsResourceWatcher) watcher).onChanged((LdsUpdate) update); + break; + case RDS: + ((RdsResourceWatcher) watcher).onChanged((RdsUpdate) update); + break; + case CDS: + ((CdsResourceWatcher) watcher).onChanged((CdsUpdate) update); + break; + case EDS: + ((EdsResourceWatcher) watcher).onChanged((EdsUpdate) update); + break; + case UNKNOWN: + default: + throw new AssertionError("should never be here"); + } + } + } + + private static final class DiscoveryRequestData { + private final ResourceType resourceType; + private final Collection resourceNames; + private final String versionInfo; + private final String responseNonce; + private final Node node; + @Nullable + private final com.google.rpc.Status errorDetail; + + DiscoveryRequestData( + ResourceType resourceType, Collection resourceNames, String versionInfo, + String responseNonce, Node node, @Nullable com.google.rpc.Status errorDetail) { + this.resourceType = resourceType; + this.resourceNames = resourceNames; + this.versionInfo = versionInfo; + this.responseNonce = responseNonce; + this.node = node; + this.errorDetail = errorDetail; + } + + DiscoveryRequest toEnvoyProto() { + DiscoveryRequest.Builder builder = + DiscoveryRequest.newBuilder() + .setVersionInfo(versionInfo) + .setNode(node.toEnvoyProtoNode()) + .addAllResourceNames(resourceNames) + .setTypeUrl(resourceType.typeUrl()) + .setResponseNonce(responseNonce); + if (errorDetail != null) { + builder.setErrorDetail(errorDetail); + } + return builder.build(); + } + + io.envoyproxy.envoy.api.v2.DiscoveryRequest toEnvoyProtoV2() { + io.envoyproxy.envoy.api.v2.DiscoveryRequest.Builder builder = + io.envoyproxy.envoy.api.v2.DiscoveryRequest.newBuilder() + .setVersionInfo(versionInfo) + .setNode(node.toEnvoyProtoNodeV2()) + .addAllResourceNames(resourceNames) + .setTypeUrl(resourceType.typeUrlV2()) + .setResponseNonce(responseNonce); + if (errorDetail != null) { + builder.setErrorDetail(errorDetail); + } + return builder.build(); + } + } + + private static final class DiscoveryResponseData { + private final ResourceType resourceType; + private final List resources; + private final String versionInfo; + private final String nonce; + + DiscoveryResponseData( + ResourceType resourceType, List resources, String versionInfo, String nonce) { + this.resourceType = resourceType; + this.resources = resources; + this.versionInfo = versionInfo; + this.nonce = nonce; + } + + ResourceType getResourceType() { + return resourceType; + } + + List getResourcesList() { + return resources; + } + + String getVersionInfo() { + return versionInfo; + } + + String getNonce() { + return nonce; + } + + static DiscoveryResponseData fromEnvoyProto(DiscoveryResponse proto) { + return new DiscoveryResponseData( + ResourceType.fromTypeUrl(proto.getTypeUrl()), proto.getResourcesList(), + proto.getVersionInfo(), proto.getNonce()); + } + + static DiscoveryResponseData fromEnvoyProtoV2( + io.envoyproxy.envoy.api.v2.DiscoveryResponse proto) { + return new DiscoveryResponseData( + ResourceType.fromTypeUrl(proto.getTypeUrl()), proto.getResourcesList(), + proto.getVersionInfo(), proto.getNonce()); + } + } + + private abstract class AbstractAdsStream { + private boolean responseReceived; + private boolean closed; + + // Response nonce for the most recently received discovery responses of each resource type. + // Client initiated requests start response nonce with empty string. + // A nonce is used to indicate the specific DiscoveryResponse each DiscoveryRequest + // corresponds to. + // A nonce becomes stale following a newer nonce being presented to the client in a + // DiscoveryResponse. + private String ldsRespNonce = ""; + private String rdsRespNonce = ""; + private String cdsRespNonce = ""; + private String edsRespNonce = ""; + + abstract void start(); + + abstract void sendDiscoveryRequest(DiscoveryRequestData request); + + abstract void sendError(Exception error); + + // Must run in syncContext. + final void handleResponse(DiscoveryResponseData response) { + if (closed) { + return; + } + responseReceived = true; + String respNonce = response.getNonce(); + // Nonce in each response is echoed back in the following ACK/NACK request. It is + // used for management server to identify which response the client is ACKing/NACking. + // To avoid confusion, client-initiated requests will always use the nonce in + // most recently received responses of each resource type. + ResourceType resourceType = response.getResourceType(); + switch (resourceType) { + case LDS: + ldsRespNonce = respNonce; + handleLdsResponse(response); + break; + case RDS: + rdsRespNonce = respNonce; + handleRdsResponse(response); + break; + case CDS: + cdsRespNonce = respNonce; + handleCdsResponse(response); + break; + case EDS: + edsRespNonce = respNonce; + handleEdsResponse(response); + break; + case UNKNOWN: + logger.log( + XdsLogLevel.WARNING, + "Received an unknown type of DiscoveryResponse\n{0}", + respNonce); + break; + default: + throw new AssertionError("Missing case in enum switch: " + resourceType); + } + } + + // Must run in syncContext. + final void handleRpcError(Throwable t) { + handleStreamClosed(Status.fromThrowable(t)); + } + + // Must run in syncContext. + final void handleRpcCompleted() { + handleStreamClosed(Status.UNAVAILABLE.withDescription("Closed by server")); + } + + private void handleStreamClosed(Status error) { + checkArgument(!error.isOk(), "unexpected OK status"); + if (closed) { + return; + } + logger.log( + XdsLogLevel.ERROR, + "ADS stream closed with status {0}: {1}. Cause: {2}", + error.getCode(), error.getDescription(), error.getCause()); + closed = true; + if (listenerWatcher != null) { + listenerWatcher.onError(error); + } + for (ResourceSubscriber subscriber : ldsResourceSubscribers.values()) { + subscriber.onError(error); + } + for (ResourceSubscriber subscriber : rdsResourceSubscribers.values()) { + subscriber.onError(error); + } + for (ResourceSubscriber subscriber : cdsResourceSubscribers.values()) { + subscriber.onError(error); + } + for (ResourceSubscriber subscriber : edsResourceSubscribers.values()) { + subscriber.onError(error); + } + cleanUp(); + cleanUpResourceTimers(); + if (responseReceived || retryBackoffPolicy == null) { + // Reset the backoff sequence if had received a response, or backoff sequence + // has never been initialized. + retryBackoffPolicy = backoffPolicyProvider.get(); + } + long delayNanos = 0; + if (!responseReceived) { + delayNanos = + Math.max( + 0, + retryBackoffPolicy.nextBackoffNanos() + - adsStreamRetryStopwatch.elapsed(TimeUnit.NANOSECONDS)); + } + logger.log(XdsLogLevel.INFO, "Retry ADS stream in {0} ns", delayNanos); + rpcRetryTimer = + syncContext.schedule( + new RpcRetryTask(), delayNanos, TimeUnit.NANOSECONDS, timeService); + } + + private void close(Exception error) { + if (closed) { + return; + } + closed = true; + cleanUp(); + sendError(error); + } + + private void cleanUp() { + if (adsStream == this) { + adsStream = null; + } + } + + /** + * Sends a DiscoveryRequest for the given resource name to management server. Memories the + * requested resource name (except for LDS as we always request for the singleton Listener) + * as we need it to find resources in responses. + */ + private void sendXdsRequest(ResourceType resourceType, Collection resourceNames) { + String version; + String nonce; + switch (resourceType) { + case LDS: + version = ldsVersion; + nonce = ldsRespNonce; + logger.log(XdsLogLevel.INFO, "Sending LDS request for resources: {0}", resourceNames); + break; + case RDS: + version = rdsVersion; + nonce = rdsRespNonce; + logger.log(XdsLogLevel.INFO, "Sending RDS request for resources: {0}", resourceNames); + break; + case CDS: + version = cdsVersion; + nonce = cdsRespNonce; + logger.log(XdsLogLevel.INFO, "Sending CDS request for resources: {0}", resourceNames); + break; + case EDS: + version = edsVersion; + nonce = edsRespNonce; + logger.log(XdsLogLevel.INFO, "Sending EDS request for resources: {0}", resourceNames); + break; + case UNKNOWN: + default: + throw new AssertionError("Unknown or missing case in enum switch: " + resourceType); + } + DiscoveryRequestData request = + new DiscoveryRequestData(resourceType, resourceNames, version, nonce, node, null); + sendDiscoveryRequest(request); + } + + /** + * Sends a DiscoveryRequest with the given information as an ACK. Updates the latest accepted + * version for the corresponding resource type. + */ + private void sendAckRequest(ResourceType resourceType, Collection resourceNames, + String versionInfo) { + String nonce; + switch (resourceType) { + case LDS: + ldsVersion = versionInfo; + nonce = ldsRespNonce; + logger.log(XdsLogLevel.WARNING, "Sending ACK for LDS update, version: {0}", versionInfo); + break; + case RDS: + rdsVersion = versionInfo; + nonce = rdsRespNonce; + logger.log(XdsLogLevel.WARNING, "Sending ACK for RDS update, version: {0}", versionInfo); + break; + case CDS: + cdsVersion = versionInfo; + nonce = cdsRespNonce; + logger.log(XdsLogLevel.WARNING, "Sending ACK for CDS update, version: {0}", versionInfo); + break; + case EDS: + edsVersion = versionInfo; + nonce = edsRespNonce; + logger.log(XdsLogLevel.WARNING, "Sending ACK for EDS update, version: {0}", versionInfo); + break; + case UNKNOWN: + default: + throw new AssertionError("Unknown or missing case in enum switch: " + resourceType); + } + DiscoveryRequestData request = + new DiscoveryRequestData(resourceType, resourceNames, versionInfo, nonce, node, null); + sendDiscoveryRequest(request); + } + + /** + * Sends a DiscoveryRequest with the given information as an NACK. NACK takes the previous + * accepted version. + */ + private void sendNackRequest(ResourceType resourceType, Collection resourceNames, + String rejectVersion, String message) { + String versionInfo; + String nonce; + switch (resourceType) { + case LDS: + versionInfo = ldsVersion; + nonce = ldsRespNonce; + logger.log( + XdsLogLevel.WARNING, + "Sending NACK for LDS update, version: {0}, reason: {1}", + rejectVersion, + message); + break; + case RDS: + versionInfo = rdsVersion; + nonce = rdsRespNonce; + logger.log( + XdsLogLevel.WARNING, + "Sending NACK for RDS update, version: {0}, reason: {1}", + rejectVersion, + message); + break; + case CDS: + versionInfo = cdsVersion; + nonce = cdsRespNonce; + logger.log( + XdsLogLevel.WARNING, + "Sending NACK for CDS update, version: {0}, reason: {1}", + rejectVersion, + message); + break; + case EDS: + versionInfo = edsVersion; + nonce = edsRespNonce; + logger.log( + XdsLogLevel.WARNING, + "Sending NACK for EDS update, version: {0}, reason: {1}", + rejectVersion, + message); + break; + case UNKNOWN: + default: + throw new AssertionError("Unknown or missing case in enum switch: " + resourceType); + } + com.google.rpc.Status error = com.google.rpc.Status.newBuilder() + .setCode(Code.INVALID_ARGUMENT_VALUE) + .setMessage(message) + .build(); + DiscoveryRequestData request = + new DiscoveryRequestData(resourceType, resourceNames, versionInfo, nonce, node, error); + sendDiscoveryRequest(request); + } + } + + private final class AdsStreamV2 extends AbstractAdsStream { + private final io.envoyproxy.envoy.service.discovery.v2.AggregatedDiscoveryServiceGrpc + .AggregatedDiscoveryServiceStub stubV2; + private StreamObserver requestWriterV2; + + AdsStreamV2() { + stubV2 = io.envoyproxy.envoy.service.discovery.v2.AggregatedDiscoveryServiceGrpc.newStub( + xdsChannel.getManagedChannel()); + } + + @Override + void start() { + StreamObserver responseReaderV2 = + new StreamObserver() { + @Override + public void onNext(final io.envoyproxy.envoy.api.v2.DiscoveryResponse response) { + syncContext.execute(new Runnable() { + @Override + public void run() { + if (logger.isLoggable(XdsLogLevel.DEBUG)) { + logger.log(XdsLogLevel.DEBUG, "Received {0} response:\n{1}", + ResourceType.fromTypeUrl(response.getTypeUrl()), + respPrinter.print(response)); + } + DiscoveryResponseData responseData = + DiscoveryResponseData.fromEnvoyProtoV2(response); + handleResponse(responseData); + } + }); + } + + @Override + public void onError(final Throwable t) { + syncContext.execute(new Runnable() { + @Override + public void run() { + handleRpcError(t); + } + }); + } + + @Override + public void onCompleted() { + syncContext.execute(new Runnable() { + @Override + public void run() { + handleRpcCompleted(); + } + }); + } + }; + requestWriterV2 = stubV2.withWaitForReady().streamAggregatedResources(responseReaderV2); + } + + @Override + void sendDiscoveryRequest(DiscoveryRequestData request) { + checkState(requestWriterV2 != null, "ADS stream has not been started"); + io.envoyproxy.envoy.api.v2.DiscoveryRequest requestProto = + request.toEnvoyProtoV2(); + requestWriterV2.onNext(requestProto); + logger.log(XdsLogLevel.DEBUG, "Sent DiscoveryRequest\n{0}", requestProto); + } + + @Override + void sendError(Exception error) { + requestWriterV2.onError(error); + } + } + + // AdsStream V3 + private final class AdsStream extends AbstractAdsStream { + private final AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceStub stub; + private StreamObserver requestWriter; + + AdsStream() { + stub = AggregatedDiscoveryServiceGrpc.newStub(xdsChannel.getManagedChannel()); + } + + @Override + void start() { + StreamObserver responseReader = new StreamObserver() { + @Override + public void onNext(final DiscoveryResponse response) { + syncContext.execute(new Runnable() { + @Override + public void run() { + if (logger.isLoggable(XdsLogLevel.DEBUG)) { + logger.log(XdsLogLevel.DEBUG, "Received {0} response:\n{1}", + ResourceType.fromTypeUrl(response.getTypeUrl()), respPrinter.print(response)); + } + DiscoveryResponseData responseData = DiscoveryResponseData.fromEnvoyProto(response); + handleResponse(responseData); + } + }); + } + + @Override + public void onError(final Throwable t) { + syncContext.execute(new Runnable() { + @Override + public void run() { + handleRpcError(t); + } + }); + } + + @Override + public void onCompleted() { + syncContext.execute(new Runnable() { + @Override + public void run() { + handleRpcCompleted(); + } + }); + } + }; + requestWriter = stub.withWaitForReady().streamAggregatedResources(responseReader); + } + + @Override + void sendDiscoveryRequest(DiscoveryRequestData request) { + checkState(requestWriter != null, "ADS stream has not been started"); + DiscoveryRequest requestProto = request.toEnvoyProto(); + requestWriter.onNext(requestProto); + logger.log(XdsLogLevel.DEBUG, "Sent DiscoveryRequest\n{0}", requestProto); + } + + @Override + void sendError(Exception error) { + requestWriter.onError(error); + } + } + + // TODO(chengyuanzhang): delete me. + @VisibleForTesting + final class ListenerResourceFetchTimeoutTask implements Runnable { + private String resourceName; + + ListenerResourceFetchTimeoutTask(String resourceName) { + this.resourceName = resourceName; + } + + @Override + public void run() { + logger.log( + XdsLogLevel.WARNING, + "Did not receive resource info {0} after {1} seconds, conclude it absent", + resourceName, INITIAL_RESOURCE_FETCH_TIMEOUT_SEC); + ldsRespTimer = null; + listenerWatcher.onResourceDoesNotExist(resourceName); + } + } + + /** + * Convert protobuf message to human readable String format. Useful for protobuf messages + * containing {@link com.google.protobuf.Any} fields. + */ + @VisibleForTesting + static final class MessagePrinter { + private final JsonFormat.Printer printer; + + @VisibleForTesting + MessagePrinter() { + com.google.protobuf.TypeRegistry registry = + com.google.protobuf.TypeRegistry.newBuilder() + .add(Listener.getDescriptor()) + .add(io.envoyproxy.envoy.api.v2.Listener.getDescriptor()) + .add(HttpConnectionManager.getDescriptor()) + .add( + io.envoyproxy.envoy.config.filter.network.http_connection_manager.v2 + .HttpConnectionManager.getDescriptor()) + .add(RouteConfiguration.getDescriptor()) + .add(io.envoyproxy.envoy.api.v2.RouteConfiguration.getDescriptor()) + .add(Cluster.getDescriptor()) + .add(io.envoyproxy.envoy.api.v2.Cluster.getDescriptor()) + .add(ClusterLoadAssignment.getDescriptor()) + .add(io.envoyproxy.envoy.api.v2.ClusterLoadAssignment.getDescriptor()) + .build(); + printer = JsonFormat.printer().usingTypeRegistry(registry); + } + + @VisibleForTesting + String print(MessageOrBuilder message) { + String res; + try { + res = printer.print(message); + } catch (InvalidProtocolBufferException e) { + res = message + " (failed to pretty-print: " + e + ")"; + } + return res; + } + } +} diff --git a/xds/src/test/java/io/grpc/xds/XdsClientImplTest2.java b/xds/src/test/java/io/grpc/xds/XdsClientImplTest2.java new file mode 100644 index 00000000000..d1bc5ee9908 --- /dev/null +++ b/xds/src/test/java/io/grpc/xds/XdsClientImplTest2.java @@ -0,0 +1,1859 @@ +/* + * Copyright 2019 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.xds; + +import static com.google.common.truth.Truth.assertThat; +import static io.grpc.xds.XdsClientTestHelper.buildCluster; +import static io.grpc.xds.XdsClientTestHelper.buildClusterLoadAssignment; +import static io.grpc.xds.XdsClientTestHelper.buildDiscoveryRequest; +import static io.grpc.xds.XdsClientTestHelper.buildDiscoveryResponse; +import static io.grpc.xds.XdsClientTestHelper.buildDropOverload; +import static io.grpc.xds.XdsClientTestHelper.buildLbEndpoint; +import static io.grpc.xds.XdsClientTestHelper.buildListener; +import static io.grpc.xds.XdsClientTestHelper.buildLocalityLbEndpoints; +import static io.grpc.xds.XdsClientTestHelper.buildRouteConfiguration; +import static io.grpc.xds.XdsClientTestHelper.buildSecureCluster; +import static io.grpc.xds.XdsClientTestHelper.buildUpstreamTlsContext; +import static io.grpc.xds.XdsClientTestHelper.buildVirtualHost; +import static io.grpc.xds.XdsClientTestHelper.buildVirtualHosts; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.protobuf.Any; +import com.google.protobuf.util.Durations; +import io.envoyproxy.envoy.config.core.v3.AggregatedConfigSource; +import io.envoyproxy.envoy.config.core.v3.ConfigSource; +import io.envoyproxy.envoy.config.core.v3.HealthStatus; +import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment; +import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment.Policy; +import io.envoyproxy.envoy.config.endpoint.v3.ClusterStats; +import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager; +import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.Rds; +import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.SdsSecretConfig; +import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext; +import io.envoyproxy.envoy.service.discovery.v3.AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceImplBase; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse; +import io.envoyproxy.envoy.service.load_stats.v3.LoadReportingServiceGrpc.LoadReportingServiceImplBase; +import io.envoyproxy.envoy.service.load_stats.v3.LoadStatsRequest; +import io.envoyproxy.envoy.service.load_stats.v3.LoadStatsResponse; +import io.grpc.Context; +import io.grpc.Context.CancellationListener; +import io.grpc.ManagedChannel; +import io.grpc.Status; +import io.grpc.Status.Code; +import io.grpc.SynchronizationContext; +import io.grpc.inprocess.InProcessChannelBuilder; +import io.grpc.inprocess.InProcessServerBuilder; +import io.grpc.internal.BackoffPolicy; +import io.grpc.internal.FakeClock; +import io.grpc.internal.FakeClock.ScheduledTask; +import io.grpc.internal.FakeClock.TaskFilter; +import io.grpc.stub.StreamObserver; +import io.grpc.testing.GrpcCleanupRule; +import io.grpc.xds.EnvoyProtoData.DropOverload; +import io.grpc.xds.EnvoyProtoData.LbEndpoint; +import io.grpc.xds.EnvoyProtoData.Locality; +import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints; +import io.grpc.xds.EnvoyProtoData.Node; +import io.grpc.xds.XdsClient.CdsResourceWatcher; +import io.grpc.xds.XdsClient.CdsUpdate; +import io.grpc.xds.XdsClient.EdsResourceWatcher; +import io.grpc.xds.XdsClient.EdsUpdate; +import io.grpc.xds.XdsClient.LdsResourceWatcher; +import io.grpc.xds.XdsClient.LdsUpdate; +import io.grpc.xds.XdsClient.RdsResourceWatcher; +import io.grpc.xds.XdsClient.RdsUpdate; +import io.grpc.xds.XdsClient.ResourceWatcher; +import io.grpc.xds.XdsClient.XdsChannel; +import io.grpc.xds.XdsClientImpl2.MessagePrinter; +import io.grpc.xds.XdsClientImpl2.ResourceType; +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatcher; +import org.mockito.Captor; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +/** + * Tests for {@link XdsClientImpl2}. + */ +@RunWith(JUnit4.class) +public class XdsClientImplTest2 { + private static final String TARGET_NAME = "hello.googleapis.com"; + private static final String LDS_RESOURCE = "listener.googleapis.com"; + private static final String RDS_RESOURCE = "route-configuration.googleapis.com"; + private static final String CDS_RESOURCE = "cluster.googleapis.com"; + private static final String EDS_RESOURCE = "cluster-load-assignment.googleapis.com"; + private static final Node NODE = Node.newBuilder().build(); + private static final FakeClock.TaskFilter RPC_RETRY_TASK_FILTER = + new FakeClock.TaskFilter() { + @Override + public boolean shouldAccept(Runnable command) { + return command.toString().contains(XdsClientImpl2.RpcRetryTask.class.getSimpleName()); + } + }; + + private static final FakeClock.TaskFilter LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER = + new TaskFilter() { + @Override + public boolean shouldAccept(Runnable command) { + return command.toString().contains(ResourceType.LDS.toString()); + } + }; + + private static final FakeClock.TaskFilter RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER = + new TaskFilter() { + @Override + public boolean shouldAccept(Runnable command) { + return command.toString().contains(ResourceType.RDS.toString()); + } + }; + + private static final FakeClock.TaskFilter CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER = + new TaskFilter() { + @Override + public boolean shouldAccept(Runnable command) { + return command.toString().contains(ResourceType.CDS.toString()); + } + }; + + private static final FakeClock.TaskFilter EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER = + new FakeClock.TaskFilter() { + @Override + public boolean shouldAccept(Runnable command) { + return command.toString().contains(ResourceType.EDS.toString()); + } + }; + + @Rule + public final GrpcCleanupRule cleanupRule = new GrpcCleanupRule(); + + private final SynchronizationContext syncContext = new SynchronizationContext( + new Thread.UncaughtExceptionHandler() { + @Override + public void uncaughtException(Thread t, Throwable e) { + throw new AssertionError(e); + } + }); + private final FakeClock fakeClock = new FakeClock(); + private final Queue> resourceDiscoveryCalls = + new ArrayDeque<>(); + private final Queue> loadReportCalls = + new ArrayDeque<>(); + private final AtomicBoolean adsEnded = new AtomicBoolean(true); + private final AtomicBoolean lrsEnded = new AtomicBoolean(true); + + @Captor + private ArgumentCaptor ldsUpdateCaptor; + @Captor + private ArgumentCaptor rdsUpdateCaptor; + @Captor + private ArgumentCaptor cdsUpdateCaptor; + @Captor + private ArgumentCaptor edsUpdateCaptor; + @Captor + private ArgumentCaptor errorCaptor; + @Mock + private BackoffPolicy.Provider backoffPolicyProvider; + @Mock + private BackoffPolicy backoffPolicy1; + @Mock + private BackoffPolicy backoffPolicy2; + @Mock + private LdsResourceWatcher ldsResourceWatcher; + @Mock + private RdsResourceWatcher rdsResourceWatcher; + @Mock + private CdsResourceWatcher cdsResourceWatcher; + @Mock + private EdsResourceWatcher edsResourceWatcher; + + private ManagedChannel channel; + private XdsClientImpl2 xdsClient; + + @Before + public void setUp() throws IOException { + MockitoAnnotations.initMocks(this); + when(backoffPolicyProvider.get()).thenReturn(backoffPolicy1, backoffPolicy2); + when(backoffPolicy1.nextBackoffNanos()).thenReturn(10L, 100L); + when(backoffPolicy2.nextBackoffNanos()).thenReturn(20L, 200L); + + final String serverName = InProcessServerBuilder.generateName(); + AggregatedDiscoveryServiceImplBase adsServiceImpl = new AggregatedDiscoveryServiceImplBase() { + @Override + public StreamObserver streamAggregatedResources( + final StreamObserver responseObserver) { + assertThat(adsEnded.get()).isTrue(); // ensure previous call was ended + adsEnded.set(false); + @SuppressWarnings("unchecked") + StreamObserver requestObserver = mock(StreamObserver.class); + RpcCall call = + new RpcCall<>(requestObserver, responseObserver); + resourceDiscoveryCalls.offer(call); + Context.current().addListener( + new CancellationListener() { + @Override + public void cancelled(Context context) { + adsEnded.set(true); + } + }, MoreExecutors.directExecutor()); + return requestObserver; + } + }; + + LoadReportingServiceImplBase lrsServiceImpl = new LoadReportingServiceImplBase() { + @Override + public StreamObserver streamLoadStats( + StreamObserver responseObserver) { + assertThat(lrsEnded.get()).isTrue(); + lrsEnded.set(false); + @SuppressWarnings("unchecked") + StreamObserver requestObserver = mock(StreamObserver.class); + RpcCall call = + new RpcCall<>(requestObserver, responseObserver); + Context.current().addListener( + new CancellationListener() { + @Override + public void cancelled(Context context) { + lrsEnded.set(true); + } + }, MoreExecutors.directExecutor()); + loadReportCalls.offer(call); + return requestObserver; + } + }; + + cleanupRule.register( + InProcessServerBuilder + .forName(serverName) + .addService(adsServiceImpl) + .addService(lrsServiceImpl) + .directExecutor() + .build() + .start()); + channel = + cleanupRule.register(InProcessChannelBuilder.forName(serverName).directExecutor().build()); + + xdsClient = + new XdsClientImpl2( + TARGET_NAME, + new XdsChannel(channel, /* useProtocolV3= */ true), + EnvoyProtoData.Node.newBuilder().build(), + syncContext, + fakeClock.getScheduledExecutorService(), + backoffPolicyProvider, + fakeClock.getStopwatchSupplier()); + + assertThat(resourceDiscoveryCalls).isEmpty(); + assertThat(loadReportCalls).isEmpty(); + } + + @After + public void tearDown() { + xdsClient.shutdown(); + assertThat(adsEnded.get()).isTrue(); + assertThat(lrsEnded.get()).isTrue(); + assertThat(channel.isShutdown()).isTrue(); + assertThat(fakeClock.getPendingTasks()).isEmpty(); + } + + @Test + public void ldsResourceNotFound() { + RpcCall call = + startResourceWatcher(ResourceType.LDS, LDS_RESOURCE, ldsResourceWatcher); + List listeners = ImmutableList.of( + Any.pack(buildListener("bar.googleapis.com", + Any.pack( + HttpConnectionManager.newBuilder() + .setRouteConfig( + buildRouteConfiguration("route-bar.googleapis.com", buildVirtualHosts(1))) + .build())))); + DiscoveryResponse response = + buildDiscoveryResponse("0", listeners, ResourceType.LDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + + // Client sends an ACK LDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "0", LDS_RESOURCE, ResourceType.LDS.typeUrl(), "0000"))); + + verifyNoInteractions(ldsResourceWatcher); + fakeClock.forwardTime(XdsClientImpl2.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); + verify(ldsResourceWatcher).onResourceDoesNotExist(LDS_RESOURCE); + assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + } + + @Test + public void ldsResourceFound_containsVirtualHosts() { + RpcCall call = + startResourceWatcher(ResourceType.LDS, LDS_RESOURCE, ldsResourceWatcher); + List listeners = ImmutableList.of( + Any.pack(buildListener(LDS_RESOURCE, + Any.pack( + HttpConnectionManager.newBuilder() + .setRouteConfig(buildRouteConfiguration("do not care", buildVirtualHosts(2))) + .build())))); + DiscoveryResponse response = + buildDiscoveryResponse("0", listeners, ResourceType.LDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + + // Client sends an ACK LDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "0", LDS_RESOURCE, ResourceType.LDS.typeUrl(), "0000"))); + verify(ldsResourceWatcher).onChanged(ldsUpdateCaptor.capture()); + assertThat(ldsUpdateCaptor.getValue().getVirtualHosts()).hasSize(2); + assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + } + + @Test + public void ldsResourceFound_containsRdsName() { + RpcCall call = + startResourceWatcher(ResourceType.LDS, LDS_RESOURCE, ldsResourceWatcher); + List listeners = ImmutableList.of( + Any.pack(buildListener(LDS_RESOURCE, + Any.pack( + HttpConnectionManager.newBuilder() + .setRds( + Rds.newBuilder() + .setRouteConfigName(RDS_RESOURCE) + .setConfigSource( + ConfigSource.newBuilder() + .setAds(AggregatedConfigSource.getDefaultInstance()))) + .build())))); + DiscoveryResponse response = + buildDiscoveryResponse("0", listeners, ResourceType.LDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + + // Client sends an ACK LDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "0", LDS_RESOURCE, ResourceType.LDS.typeUrl(), "0000"))); + verify(ldsResourceWatcher).onChanged(ldsUpdateCaptor.capture()); + assertThat(ldsUpdateCaptor.getValue().getRdsName()).isEqualTo(RDS_RESOURCE); + assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + } + + @Test + public void cachedLdsResource_data() { + RpcCall call = + startResourceWatcher(ResourceType.LDS, LDS_RESOURCE, ldsResourceWatcher); + List listeners = ImmutableList.of( + Any.pack(buildListener(LDS_RESOURCE, + Any.pack( + HttpConnectionManager.newBuilder() + .setRds( + Rds.newBuilder() + .setRouteConfigName(RDS_RESOURCE) + .setConfigSource( + ConfigSource.newBuilder() + .setAds(AggregatedConfigSource.getDefaultInstance()))) + .build())))); + DiscoveryResponse response = + buildDiscoveryResponse("0", listeners, ResourceType.LDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + + // Client sends an ACK LDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "0", LDS_RESOURCE, ResourceType.LDS.typeUrl(), "0000"))); + LdsResourceWatcher watcher = mock(LdsResourceWatcher.class); + xdsClient.watchLdsResource(LDS_RESOURCE, watcher); + verify(watcher).onChanged(ldsUpdateCaptor.capture()); + assertThat(ldsUpdateCaptor.getValue().getRdsName()).isEqualTo(RDS_RESOURCE); + verifyNoMoreInteractions(call.requestObserver); + } + + @Test + public void cachedLdsResource_absent() { + RpcCall call = + startResourceWatcher(ResourceType.LDS, LDS_RESOURCE, ldsResourceWatcher); + fakeClock.forwardTime(XdsClientImpl2.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); + verify(ldsResourceWatcher).onResourceDoesNotExist(LDS_RESOURCE); + LdsResourceWatcher watcher = mock(LdsResourceWatcher.class); + xdsClient.watchLdsResource(LDS_RESOURCE, watcher); + verify(watcher).onResourceDoesNotExist(LDS_RESOURCE); + verifyNoMoreInteractions(call.requestObserver); + } + + @Test + public void ldsResourceUpdated() { + RpcCall call = + startResourceWatcher(ResourceType.LDS, LDS_RESOURCE, ldsResourceWatcher); + List listeners = ImmutableList.of( + Any.pack(buildListener(LDS_RESOURCE, + Any.pack( + HttpConnectionManager.newBuilder() + .setRouteConfig(buildRouteConfiguration("do not care", buildVirtualHosts(2))) + .build())))); + DiscoveryResponse response = + buildDiscoveryResponse("0", listeners, ResourceType.LDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + + // Client sends an ACK LDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "0", LDS_RESOURCE, ResourceType.LDS.typeUrl(), "0000"))); + verify(ldsResourceWatcher).onChanged(ldsUpdateCaptor.capture()); + assertThat(ldsUpdateCaptor.getValue().getVirtualHosts()).hasSize(2); + + listeners = ImmutableList.of( + Any.pack(buildListener(LDS_RESOURCE, + Any.pack( + HttpConnectionManager.newBuilder() + .setRds( + Rds.newBuilder() + .setRouteConfigName(RDS_RESOURCE) + .setConfigSource( + ConfigSource.newBuilder() + .setAds(AggregatedConfigSource.getDefaultInstance()))) + .build())))); + response = + buildDiscoveryResponse("1", listeners, ResourceType.LDS.typeUrl(), "0001"); + call.responseObserver.onNext(response); + + // Client sends an ACK LDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "1", LDS_RESOURCE, ResourceType.LDS.typeUrl(), "0001"))); + verify(ldsResourceWatcher, times(2)).onChanged(ldsUpdateCaptor.capture()); + assertThat(ldsUpdateCaptor.getValue().getRdsName()).isEqualTo(RDS_RESOURCE); + } + + @Test + public void ldsResourceDeleted() { + RpcCall call = + startResourceWatcher(ResourceType.LDS, LDS_RESOURCE, ldsResourceWatcher); + List listeners = ImmutableList.of( + Any.pack(buildListener(LDS_RESOURCE, + Any.pack( + HttpConnectionManager.newBuilder() + .setRouteConfig(buildRouteConfiguration("do not care", buildVirtualHosts(2))) + .build())))); + DiscoveryResponse response = + buildDiscoveryResponse("0", listeners, ResourceType.LDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + + // Client sends an ACK LDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "0", LDS_RESOURCE, ResourceType.LDS.typeUrl(), "0000"))); + verify(ldsResourceWatcher).onChanged(ldsUpdateCaptor.capture()); + assertThat(ldsUpdateCaptor.getValue().getVirtualHosts()).hasSize(2); + + response = buildDiscoveryResponse("1", Collections.emptyList(), + ResourceType.LDS.typeUrl(), "0001"); + call.responseObserver.onNext(response); + + // Client sends an ACK LDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "1", LDS_RESOURCE, ResourceType.LDS.typeUrl(), "0001"))); + verify(ldsResourceWatcher).onResourceDoesNotExist(LDS_RESOURCE); + } + + @Test + public void multipleLdsWatchers() { + String ldsResource = "bar.googleapis.com"; + LdsResourceWatcher watcher1 = mock(LdsResourceWatcher.class); + LdsResourceWatcher watcher2 = mock(LdsResourceWatcher.class); + xdsClient.watchLdsResource(LDS_RESOURCE, ldsResourceWatcher); + xdsClient.watchLdsResource(ldsResource, watcher1); + xdsClient.watchLdsResource(ldsResource, watcher2); + RpcCall call = resourceDiscoveryCalls.poll(); + verify(call.requestObserver).onNext( + argThat(new DiscoveryRequestMatcher(NODE, "", Arrays.asList(LDS_RESOURCE, ldsResource), + ResourceType.LDS.typeUrl(), ""))); + + fakeClock.forwardTime(XdsClientImpl2.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); + verify(ldsResourceWatcher).onResourceDoesNotExist(LDS_RESOURCE); + verify(watcher1).onResourceDoesNotExist(ldsResource); + verify(watcher2).onResourceDoesNotExist(ldsResource); + + List listeners = ImmutableList.of( + Any.pack(buildListener(LDS_RESOURCE, + Any.pack( + HttpConnectionManager.newBuilder() + .setRouteConfig(buildRouteConfiguration("do not care", buildVirtualHosts(2))) + .build()))), + Any.pack(buildListener(ldsResource, + Any.pack( + HttpConnectionManager.newBuilder() + .setRouteConfig(buildRouteConfiguration("do not care", buildVirtualHosts(4))) + .build())))); + DiscoveryResponse response = + buildDiscoveryResponse("0", listeners, ResourceType.LDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + verify(ldsResourceWatcher).onChanged(ldsUpdateCaptor.capture()); + assertThat(ldsUpdateCaptor.getValue().getVirtualHosts()).hasSize(2); + verify(watcher1).onChanged(ldsUpdateCaptor.capture()); + assertThat(ldsUpdateCaptor.getValue().getVirtualHosts()).hasSize(4); + verify(watcher2).onChanged(ldsUpdateCaptor.capture()); + assertThat(ldsUpdateCaptor.getValue().getVirtualHosts()).hasSize(4); + } + + @Test + public void rdsResourceNotFound() { + RpcCall call = + startResourceWatcher(ResourceType.RDS, RDS_RESOURCE, rdsResourceWatcher); + List routeConfigs = ImmutableList.of( + Any.pack(buildRouteConfiguration("route-bar.googleapis.com", buildVirtualHosts(2)))); + DiscoveryResponse response = + buildDiscoveryResponse("0", routeConfigs, ResourceType.RDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + + // Client sends an ACK RDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "0", RDS_RESOURCE, ResourceType.RDS.typeUrl(), "0000"))); + + verifyNoInteractions(rdsResourceWatcher); + fakeClock.forwardTime(XdsClientImpl2.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); + verify(rdsResourceWatcher).onResourceDoesNotExist(RDS_RESOURCE); + assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + } + + @Test + public void rdsResourceFound() { + RpcCall call = + startResourceWatcher(ResourceType.RDS, RDS_RESOURCE, rdsResourceWatcher); + List routeConfigs = + ImmutableList.of(Any.pack(buildRouteConfiguration(RDS_RESOURCE, buildVirtualHosts(2)))); + DiscoveryResponse response = + buildDiscoveryResponse("0", routeConfigs, ResourceType.RDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + + // Client sends an ACK RDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "0", RDS_RESOURCE, ResourceType.RDS.typeUrl(), "0000"))); + verify(rdsResourceWatcher).onChanged(rdsUpdateCaptor.capture()); + assertThat(rdsUpdateCaptor.getValue().getVirtualHosts()).hasSize(2); + assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + } + + @Test + public void cachedRdsResource_data() { + RpcCall call = + startResourceWatcher(ResourceType.RDS, RDS_RESOURCE, rdsResourceWatcher); + List routeConfigs = + ImmutableList.of(Any.pack(buildRouteConfiguration(RDS_RESOURCE, buildVirtualHosts(2)))); + DiscoveryResponse response = + buildDiscoveryResponse("0", routeConfigs, ResourceType.RDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + + // Client sends an ACK RDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "0", RDS_RESOURCE, ResourceType.RDS.typeUrl(), "0000"))); + + RdsResourceWatcher watcher = mock(RdsResourceWatcher.class); + xdsClient.watchRdsResource(RDS_RESOURCE, watcher); + verify(watcher).onChanged(rdsUpdateCaptor.capture()); + assertThat(rdsUpdateCaptor.getValue().getVirtualHosts()).hasSize(2); + verifyNoMoreInteractions(call.requestObserver); + } + + @Test + public void cachedRdsResource_absent() { + RpcCall call = + startResourceWatcher(ResourceType.RDS, RDS_RESOURCE, rdsResourceWatcher); + fakeClock.forwardTime(XdsClientImpl2.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); + verify(rdsResourceWatcher).onResourceDoesNotExist(RDS_RESOURCE); + RdsResourceWatcher watcher = mock(RdsResourceWatcher.class); + xdsClient.watchRdsResource(RDS_RESOURCE, watcher); + verify(watcher).onResourceDoesNotExist(RDS_RESOURCE); + verifyNoMoreInteractions(call.requestObserver); + } + + @Test + public void rdsResourceUpdated() { + RpcCall call = + startResourceWatcher(ResourceType.RDS, RDS_RESOURCE, rdsResourceWatcher); + List routeConfigs = + ImmutableList.of(Any.pack(buildRouteConfiguration(RDS_RESOURCE, buildVirtualHosts(2)))); + DiscoveryResponse response = + buildDiscoveryResponse("0", routeConfigs, ResourceType.RDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + + // Client sends an ACK RDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "0", RDS_RESOURCE, ResourceType.RDS.typeUrl(), "0000"))); + verify(rdsResourceWatcher).onChanged(rdsUpdateCaptor.capture()); + assertThat(rdsUpdateCaptor.getValue().getVirtualHosts()).hasSize(2); + + routeConfigs = + ImmutableList.of(Any.pack(buildRouteConfiguration(RDS_RESOURCE, buildVirtualHosts(4)))); + response = + buildDiscoveryResponse("1", routeConfigs, ResourceType.RDS.typeUrl(), "0001"); + call.responseObserver.onNext(response); + + // Client sends an ACK RDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "1", RDS_RESOURCE, ResourceType.RDS.typeUrl(), "0001"))); + verify(rdsResourceWatcher, times(2)).onChanged(rdsUpdateCaptor.capture()); + assertThat(rdsUpdateCaptor.getValue().getVirtualHosts()).hasSize(4); + } + + @Test + public void rdsResourceDeletedByLds() { + xdsClient.watchLdsResource(LDS_RESOURCE, ldsResourceWatcher); + xdsClient.watchRdsResource(RDS_RESOURCE, rdsResourceWatcher); + RpcCall call = resourceDiscoveryCalls.poll(); + List listeners = ImmutableList.of( + Any.pack(buildListener(LDS_RESOURCE, + Any.pack( + HttpConnectionManager.newBuilder() + .setRds( + Rds.newBuilder() + .setRouteConfigName(RDS_RESOURCE) + .setConfigSource( + ConfigSource.newBuilder() + .setAds(AggregatedConfigSource.getDefaultInstance()))) + .build())))); + DiscoveryResponse response = + buildDiscoveryResponse("0", listeners, ResourceType.LDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + verify(ldsResourceWatcher).onChanged(ldsUpdateCaptor.capture()); + assertThat(ldsUpdateCaptor.getValue().getRdsName()).isEqualTo(RDS_RESOURCE); + + List routeConfigs = + ImmutableList.of(Any.pack(buildRouteConfiguration(RDS_RESOURCE, buildVirtualHosts(2)))); + response = buildDiscoveryResponse("0", routeConfigs, ResourceType.RDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + verify(rdsResourceWatcher).onChanged(rdsUpdateCaptor.capture()); + assertThat(rdsUpdateCaptor.getValue().getVirtualHosts()).hasSize(2); + + listeners = ImmutableList.of( + Any.pack(buildListener(LDS_RESOURCE, + Any.pack( + HttpConnectionManager.newBuilder() + .setRouteConfig(buildRouteConfiguration("do not care", buildVirtualHosts(5))) + .build())))); + response = buildDiscoveryResponse("1", listeners, ResourceType.LDS.typeUrl(), "0001"); + call.responseObserver.onNext(response); + verify(ldsResourceWatcher, times(2)).onChanged(ldsUpdateCaptor.capture()); + assertThat(ldsUpdateCaptor.getValue().getVirtualHosts()).hasSize(5); + verify(rdsResourceWatcher).onResourceDoesNotExist(RDS_RESOURCE); + } + + @Test + public void multipleRdsWatchers() { + String rdsResource = "route-bar.googleapis.com"; + RdsResourceWatcher watcher1 = mock(RdsResourceWatcher.class); + RdsResourceWatcher watcher2 = mock(RdsResourceWatcher.class); + xdsClient.watchRdsResource(RDS_RESOURCE, rdsResourceWatcher); + xdsClient.watchRdsResource(rdsResource, watcher1); + xdsClient.watchRdsResource(rdsResource, watcher2); + RpcCall call = resourceDiscoveryCalls.poll(); + verify(call.requestObserver).onNext( + argThat(new DiscoveryRequestMatcher(NODE, "", Arrays.asList(RDS_RESOURCE, rdsResource), + ResourceType.RDS.typeUrl(), ""))); + + fakeClock.forwardTime(XdsClientImpl2.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); + verify(rdsResourceWatcher).onResourceDoesNotExist(RDS_RESOURCE); + verify(watcher1).onResourceDoesNotExist(rdsResource); + verify(watcher2).onResourceDoesNotExist(rdsResource); + + List routeConfigs = + ImmutableList.of(Any.pack(buildRouteConfiguration(RDS_RESOURCE, buildVirtualHosts(2)))); + DiscoveryResponse response = + buildDiscoveryResponse("0", routeConfigs, ResourceType.RDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + + verify(rdsResourceWatcher).onChanged(rdsUpdateCaptor.capture()); + assertThat(rdsUpdateCaptor.getValue().getVirtualHosts()).hasSize(2); + verifyNoMoreInteractions(watcher1, watcher2); + + routeConfigs = + ImmutableList.of(Any.pack(buildRouteConfiguration(rdsResource, buildVirtualHosts(4)))); + response = + buildDiscoveryResponse("2", routeConfigs, ResourceType.RDS.typeUrl(), "0002"); + call.responseObserver.onNext(response); + + verify(watcher1).onChanged(rdsUpdateCaptor.capture()); + assertThat(rdsUpdateCaptor.getValue().getVirtualHosts()).hasSize(4); + verify(watcher2).onChanged(rdsUpdateCaptor.capture()); + assertThat(rdsUpdateCaptor.getValue().getVirtualHosts()).hasSize(4); + verifyNoMoreInteractions(rdsResourceWatcher); + } + + @Test + public void cdsResourceNotFound() { + RpcCall call = + startResourceWatcher(ResourceType.CDS, CDS_RESOURCE, cdsResourceWatcher); + + List clusters = ImmutableList.of( + Any.pack(buildCluster("cluster-bar.googleapis.com", null, false)), + Any.pack(buildCluster("cluster-baz.googleapis.com", null, false))); + DiscoveryResponse response = + buildDiscoveryResponse("0", clusters, ResourceType.CDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + + // Client sent an ACK CDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "0", CDS_RESOURCE, ResourceType.CDS.typeUrl(), "0000"))); + verifyNoInteractions(cdsResourceWatcher); + + fakeClock.forwardTime(XdsClientImpl2.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); + verify(cdsResourceWatcher).onResourceDoesNotExist(CDS_RESOURCE); + assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + } + + @Test + public void cdsResourceFound() { + RpcCall call = + startResourceWatcher(ResourceType.CDS, CDS_RESOURCE, cdsResourceWatcher); + List clusters = ImmutableList.of(Any.pack(buildCluster(CDS_RESOURCE, null, false))); + DiscoveryResponse response = + buildDiscoveryResponse("0", clusters, ResourceType.CDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + + // Client sent an ACK CDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "0", CDS_RESOURCE, ResourceType.CDS.typeUrl(), "0000"))); + verify(cdsResourceWatcher).onChanged(cdsUpdateCaptor.capture()); + CdsUpdate cdsUpdate = cdsUpdateCaptor.getValue(); + assertThat(cdsUpdate.getClusterName()).isEqualTo(CDS_RESOURCE); + assertThat(cdsUpdate.getEdsServiceName()).isNull(); + assertThat(cdsUpdate.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate.getLrsServerName()).isNull(); + assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + } + + /** + * CDS response containing UpstreamTlsContext for a cluster. + */ + @Test + public void cdsResponseWithUpstreamTlsContext() { + RpcCall call = + startResourceWatcher(ResourceType.CDS, CDS_RESOURCE, cdsResourceWatcher); + + // Management server sends back CDS response with UpstreamTlsContext. + UpstreamTlsContext testUpstreamTlsContext = + buildUpstreamTlsContext("secret1", "unix:/var/uds2"); + List clusters = ImmutableList.of( + Any.pack(buildCluster("cluster-bar.googleapis.com", null, false)), + Any.pack(buildSecureCluster(CDS_RESOURCE, + "eds-cluster-foo.googleapis.com", true, testUpstreamTlsContext)), + Any.pack(buildCluster("cluster-baz.googleapis.com", null, false))); + DiscoveryResponse response = + buildDiscoveryResponse("0", clusters, ResourceType.CDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + + // Client sent an ACK CDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "0", CDS_RESOURCE, ResourceType.CDS.typeUrl(), "0000"))); + verify(cdsResourceWatcher, times(1)).onChanged(cdsUpdateCaptor.capture()); + CdsUpdate cdsUpdate = cdsUpdateCaptor.getValue(); + EnvoyServerProtoData.UpstreamTlsContext upstreamTlsContext = cdsUpdate + .getUpstreamTlsContext(); + SdsSecretConfig validationContextSdsSecretConfig = upstreamTlsContext.getCommonTlsContext() + .getValidationContextSdsSecretConfig(); + assertThat(validationContextSdsSecretConfig.getName()).isEqualTo("secret1"); + assertThat( + Iterables.getOnlyElement( + validationContextSdsSecretConfig + .getSdsConfig() + .getApiConfigSource() + .getGrpcServicesList()) + .getGoogleGrpc() + .getTargetUri()) + .isEqualTo("unix:/var/uds2"); + } + + @Test + public void cachedCdsResource_data() { + RpcCall call = + startResourceWatcher(ResourceType.CDS, CDS_RESOURCE, cdsResourceWatcher); + List clusters = ImmutableList.of(Any.pack(buildCluster(CDS_RESOURCE, null, false))); + DiscoveryResponse response = + buildDiscoveryResponse("0", clusters, ResourceType.CDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + + // Client sends an ACK CDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "0", CDS_RESOURCE, ResourceType.CDS.typeUrl(), "0000"))); + + CdsResourceWatcher watcher = mock(CdsResourceWatcher.class); + xdsClient.watchCdsResource(CDS_RESOURCE, watcher); + verify(watcher).onChanged(cdsUpdateCaptor.capture()); + CdsUpdate cdsUpdate = cdsUpdateCaptor.getValue(); + assertThat(cdsUpdate.getClusterName()).isEqualTo(CDS_RESOURCE); + assertThat(cdsUpdate.getEdsServiceName()).isNull(); + assertThat(cdsUpdate.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate.getLrsServerName()).isNull(); + verifyNoMoreInteractions(call.requestObserver); + } + + @Test + public void cachedCdsResource_absent() { + RpcCall call = + startResourceWatcher(ResourceType.CDS, CDS_RESOURCE, cdsResourceWatcher); + fakeClock.forwardTime(XdsClientImpl2.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); + verify(cdsResourceWatcher).onResourceDoesNotExist(CDS_RESOURCE); + CdsResourceWatcher watcher = mock(CdsResourceWatcher.class); + xdsClient.watchCdsResource(CDS_RESOURCE, watcher); + verify(watcher).onResourceDoesNotExist(CDS_RESOURCE); + verifyNoMoreInteractions(call.requestObserver); + } + + @Test + public void cdsResourceUpdated() { + RpcCall call = + startResourceWatcher(ResourceType.CDS, CDS_RESOURCE, cdsResourceWatcher); + List clusters = ImmutableList.of(Any.pack(buildCluster(CDS_RESOURCE, null, false))); + DiscoveryResponse response = + buildDiscoveryResponse("0", clusters, ResourceType.CDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + + // Client sends an ACK CDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "0", CDS_RESOURCE, ResourceType.CDS.typeUrl(), "0000"))); + verify(cdsResourceWatcher).onChanged(cdsUpdateCaptor.capture()); + CdsUpdate cdsUpdate = cdsUpdateCaptor.getValue(); + assertThat(cdsUpdate.getClusterName()).isEqualTo(CDS_RESOURCE); + assertThat(cdsUpdate.getEdsServiceName()).isNull(); + assertThat(cdsUpdate.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate.getLrsServerName()).isNull(); + + String edsService = "eds-service-bar.googleapis.com"; + clusters = ImmutableList.of(Any.pack(buildCluster(CDS_RESOURCE, edsService, true))); + response = buildDiscoveryResponse("1", clusters, ResourceType.CDS.typeUrl(), "0001"); + call.responseObserver.onNext(response); + + // Client sends an ACK CDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "1", CDS_RESOURCE, ResourceType.CDS.typeUrl(), "0001"))); + verify(cdsResourceWatcher, times(2)).onChanged(cdsUpdateCaptor.capture()); + cdsUpdate = cdsUpdateCaptor.getValue(); + assertThat(cdsUpdate.getClusterName()).isEqualTo(CDS_RESOURCE); + assertThat(cdsUpdate.getEdsServiceName()).isEqualTo(edsService); + assertThat(cdsUpdate.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate.getLrsServerName()).isEqualTo(""); + } + + @Test + public void cdsResourceDeleted() { + RpcCall call = + startResourceWatcher(ResourceType.CDS, CDS_RESOURCE, cdsResourceWatcher); + List clusters = ImmutableList.of(Any.pack(buildCluster(CDS_RESOURCE, null, false))); + DiscoveryResponse response = + buildDiscoveryResponse("0", clusters, ResourceType.CDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + + // Client sends an ACK CDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "0", CDS_RESOURCE, ResourceType.CDS.typeUrl(), "0000"))); + verify(cdsResourceWatcher).onChanged(cdsUpdateCaptor.capture()); + CdsUpdate cdsUpdate = cdsUpdateCaptor.getValue(); + assertThat(cdsUpdate.getClusterName()).isEqualTo(CDS_RESOURCE); + assertThat(cdsUpdate.getEdsServiceName()).isNull(); + assertThat(cdsUpdate.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate.getLrsServerName()).isNull(); + + response = buildDiscoveryResponse("1", Collections.emptyList(), + ResourceType.CDS.typeUrl(), "0001"); + call.responseObserver.onNext(response); + + // Client sends an ACK CDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "1", CDS_RESOURCE, ResourceType.CDS.typeUrl(), "0001"))); + verify(cdsResourceWatcher).onResourceDoesNotExist(CDS_RESOURCE); + } + + @Test + public void multipleCdsWatchers() { + String cdsResource = "cluster-bar.googleapis.com"; + CdsResourceWatcher watcher1 = mock(CdsResourceWatcher.class); + CdsResourceWatcher watcher2 = mock(CdsResourceWatcher.class); + xdsClient.watchCdsResource(CDS_RESOURCE, cdsResourceWatcher); + xdsClient.watchCdsResource(cdsResource, watcher1); + xdsClient.watchCdsResource(cdsResource, watcher2); + RpcCall call = resourceDiscoveryCalls.poll(); + verify(call.requestObserver).onNext( + argThat(new DiscoveryRequestMatcher(NODE, "", Arrays.asList(CDS_RESOURCE, cdsResource), + ResourceType.CDS.typeUrl(), ""))); + + fakeClock.forwardTime(XdsClientImpl2.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); + verify(cdsResourceWatcher).onResourceDoesNotExist(CDS_RESOURCE); + verify(watcher1).onResourceDoesNotExist(cdsResource); + verify(watcher2).onResourceDoesNotExist(cdsResource); + + String edsService = "eds-service-bar.googleapis.com"; + List clusters = ImmutableList.of( + Any.pack(buildCluster(CDS_RESOURCE, null, false)), + Any.pack(buildCluster(cdsResource, edsService, true))); + DiscoveryResponse response = + buildDiscoveryResponse("0", clusters, ResourceType.CDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + verify(cdsResourceWatcher).onChanged(cdsUpdateCaptor.capture()); + CdsUpdate cdsUpdate = cdsUpdateCaptor.getValue(); + assertThat(cdsUpdate.getClusterName()).isEqualTo(CDS_RESOURCE); + assertThat(cdsUpdate.getEdsServiceName()).isNull(); + assertThat(cdsUpdate.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate.getLrsServerName()).isNull(); + verify(watcher1).onChanged(cdsUpdateCaptor.capture()); + cdsUpdate = cdsUpdateCaptor.getValue(); + assertThat(cdsUpdate.getClusterName()).isEqualTo(cdsResource); + assertThat(cdsUpdate.getEdsServiceName()).isEqualTo(edsService); + assertThat(cdsUpdate.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate.getLrsServerName()).isEqualTo(""); + verify(watcher2).onChanged(cdsUpdateCaptor.capture()); + cdsUpdate = cdsUpdateCaptor.getValue(); + assertThat(cdsUpdate.getClusterName()).isEqualTo(cdsResource); + assertThat(cdsUpdate.getEdsServiceName()).isEqualTo(edsService); + assertThat(cdsUpdate.getLbPolicy()).isEqualTo("round_robin"); + assertThat(cdsUpdate.getLrsServerName()).isEqualTo(""); + } + + @Test + public void edsResourceNotFound() { + RpcCall call = + startResourceWatcher(ResourceType.EDS, EDS_RESOURCE, edsResourceWatcher); + List clusterLoadAssignments = + ImmutableList.of( + Any.pack( + buildClusterLoadAssignment("cluster-bar.googleapis.com", + ImmutableList.of( + buildLocalityLbEndpoints("region1", "zone1", "subzone1", + ImmutableList.of( + buildLbEndpoint("192.168.0.1", 8080, HealthStatus.HEALTHY, 2)), + 1, 0)), + ImmutableList.of()))); + DiscoveryResponse response = + buildDiscoveryResponse("0", clusterLoadAssignments, ResourceType.EDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + + // Client sent an ACK EDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "0", EDS_RESOURCE, ResourceType.EDS.typeUrl(), "0000"))); + verifyNoInteractions(edsResourceWatcher); + + fakeClock.forwardTime(XdsClientImpl2.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); + verify(edsResourceWatcher).onResourceDoesNotExist(EDS_RESOURCE); + assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + } + + @Test + public void edsResourceFound() { + RpcCall call = + startResourceWatcher(ResourceType.EDS, EDS_RESOURCE, edsResourceWatcher); + List clusterLoadAssignments = + ImmutableList.of( + Any.pack( + buildClusterLoadAssignment(EDS_RESOURCE, + ImmutableList.of( + buildLocalityLbEndpoints("region1", "zone1", "subzone1", + ImmutableList.of( + buildLbEndpoint("192.168.0.1", 8080, HealthStatus.HEALTHY, 2)), + 1, 0), + buildLocalityLbEndpoints("region3", "zone3", "subzone3", + ImmutableList.of(), + 2, 1), /* locality with 0 endpoint */ + buildLocalityLbEndpoints("region4", "zone4", "subzone4", + ImmutableList.of( + buildLbEndpoint("192.168.142.5", 80, HealthStatus.UNKNOWN, 5)), + 0, 2) /* locality with 0 weight */), + ImmutableList.of( + buildDropOverload("lb", 200), + buildDropOverload("throttle", 1000))))); + DiscoveryResponse response = + buildDiscoveryResponse("0", clusterLoadAssignments, ResourceType.EDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + + // Client sent an ACK EDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "0", EDS_RESOURCE, ResourceType.EDS.typeUrl(), "0000"))); + verify(edsResourceWatcher).onChanged(edsUpdateCaptor.capture()); + EdsUpdate edsUpdate = edsUpdateCaptor.getValue(); + assertThat(edsUpdate.getClusterName()).isEqualTo(EDS_RESOURCE); + assertThat(edsUpdate.getDropPolicies()) + .containsExactly( + new DropOverload("lb", 200), + new DropOverload("throttle", 1000)); + assertThat(edsUpdate.getLocalityLbEndpointsMap()) + .containsExactly( + new Locality("region1", "zone1", "subzone1"), + new LocalityLbEndpoints( + ImmutableList.of( + new LbEndpoint("192.168.0.1", 8080, + 2, true)), 1, 0), + new Locality("region3", "zone3", "subzone3"), + new LocalityLbEndpoints(ImmutableList.of(), 2, 1)); + } + + @Test + public void cachedEdsResource_data() { + RpcCall call = + startResourceWatcher(ResourceType.EDS, EDS_RESOURCE, edsResourceWatcher); + List clusterLoadAssignments = + ImmutableList.of( + Any.pack( + buildClusterLoadAssignment(EDS_RESOURCE, + ImmutableList.of( + buildLocalityLbEndpoints("region1", "zone1", "subzone1", + ImmutableList.of( + buildLbEndpoint("192.168.0.1", 8080, HealthStatus.HEALTHY, 2)), + 1, 0), + buildLocalityLbEndpoints("region3", "zone3", "subzone3", + ImmutableList.of(), + 2, 1), /* locality with 0 endpoint */ + buildLocalityLbEndpoints("region4", "zone4", "subzone4", + ImmutableList.of( + buildLbEndpoint("192.168.142.5", 80, HealthStatus.UNKNOWN, 5)), + 0, 2) /* locality with 0 weight */), + ImmutableList.of( + buildDropOverload("lb", 200), + buildDropOverload("throttle", 1000))))); + DiscoveryResponse response = + buildDiscoveryResponse("0", clusterLoadAssignments, ResourceType.EDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + + // Client sends an ACK EDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "0", EDS_RESOURCE, ResourceType.EDS.typeUrl(), "0000"))); + + EdsResourceWatcher watcher = mock(EdsResourceWatcher.class); + xdsClient.watchEdsResource(EDS_RESOURCE, watcher); + verify(watcher).onChanged(edsUpdateCaptor.capture()); + EdsUpdate edsUpdate = edsUpdateCaptor.getValue(); + assertThat(edsUpdate.getClusterName()).isEqualTo(EDS_RESOURCE); + assertThat(edsUpdate.getDropPolicies()) + .containsExactly( + new DropOverload("lb", 200), + new DropOverload("throttle", 1000)); + assertThat(edsUpdate.getLocalityLbEndpointsMap()) + .containsExactly( + new Locality("region1", "zone1", "subzone1"), + new LocalityLbEndpoints( + ImmutableList.of( + new LbEndpoint("192.168.0.1", 8080, + 2, true)), 1, 0), + new Locality("region3", "zone3", "subzone3"), + new LocalityLbEndpoints(ImmutableList.of(), 2, 1)); + verifyNoMoreInteractions(call.requestObserver); + } + + @Test + public void cachedEdsResource_absent() { + RpcCall call = + startResourceWatcher(ResourceType.EDS, EDS_RESOURCE, edsResourceWatcher); + fakeClock.forwardTime(XdsClientImpl2.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); + verify(edsResourceWatcher).onResourceDoesNotExist(EDS_RESOURCE); + EdsResourceWatcher watcher = mock(EdsResourceWatcher.class); + xdsClient.watchEdsResource(EDS_RESOURCE, watcher); + verify(watcher).onResourceDoesNotExist(EDS_RESOURCE); + verifyNoMoreInteractions(call.requestObserver); + } + + @Test + public void edsResourceUpdated() { + RpcCall call = + startResourceWatcher(ResourceType.EDS, EDS_RESOURCE, edsResourceWatcher); + List clusterLoadAssignments = + ImmutableList.of( + Any.pack( + buildClusterLoadAssignment(EDS_RESOURCE, + ImmutableList.of( + buildLocalityLbEndpoints("region1", "zone1", "subzone1", + ImmutableList.of( + buildLbEndpoint("192.168.0.1", 8080, HealthStatus.HEALTHY, 2)), + 1, 0), + buildLocalityLbEndpoints("region3", "zone3", "subzone3", + ImmutableList.of(), + 2, 1), /* locality with 0 endpoint */ + buildLocalityLbEndpoints("region4", "zone4", "subzone4", + ImmutableList.of( + buildLbEndpoint("192.168.142.5", 80, HealthStatus.UNKNOWN, 5)), + 0, 2) /* locality with 0 weight */), + ImmutableList.of( + buildDropOverload("lb", 200), + buildDropOverload("throttle", 1000))))); + DiscoveryResponse response = + buildDiscoveryResponse("0", clusterLoadAssignments, ResourceType.EDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + + // Client sent an ACK EDS request. + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "0", EDS_RESOURCE, ResourceType.EDS.typeUrl(), "0000"))); + verify(edsResourceWatcher).onChanged(edsUpdateCaptor.capture()); + EdsUpdate edsUpdate = edsUpdateCaptor.getValue(); + assertThat(edsUpdate.getClusterName()).isEqualTo(EDS_RESOURCE); + assertThat(edsUpdate.getDropPolicies()) + .containsExactly( + new DropOverload("lb", 200), + new DropOverload("throttle", 1000)); + assertThat(edsUpdate.getLocalityLbEndpointsMap()) + .containsExactly( + new Locality("region1", "zone1", "subzone1"), + new LocalityLbEndpoints( + ImmutableList.of( + new LbEndpoint("192.168.0.1", 8080, 2, true)), 1, 0), + new Locality("region3", "zone3", "subzone3"), + new LocalityLbEndpoints(ImmutableList.of(), 2, 1)); + + clusterLoadAssignments = + ImmutableList.of( + Any.pack( + buildClusterLoadAssignment(EDS_RESOURCE, + ImmutableList.of( + buildLocalityLbEndpoints("region2", "zone2", "subzone2", + ImmutableList.of( + buildLbEndpoint("172.44.2.2", 8000, HealthStatus.HEALTHY, 3)), + 2, 0)), + ImmutableList.of()))); + response = + buildDiscoveryResponse("1", clusterLoadAssignments, ResourceType.EDS.typeUrl(), "0001"); + call.responseObserver.onNext(response); + + verify(edsResourceWatcher, times(2)).onChanged(edsUpdateCaptor.capture()); + edsUpdate = edsUpdateCaptor.getValue(); + assertThat(edsUpdate.getClusterName()).isEqualTo(EDS_RESOURCE); + assertThat(edsUpdate.getDropPolicies()).isEmpty(); + assertThat(edsUpdate.getLocalityLbEndpointsMap()) + .containsExactly( + new Locality("region2", "zone2", "subzone2"), + new LocalityLbEndpoints( + ImmutableList.of( + new LbEndpoint("172.44.2.2", 8000, 3, true)), 2, 0)); + } + + @Test + public void edsResourceDeletedByCds() { + xdsClient.watchCdsResource(CDS_RESOURCE, cdsResourceWatcher); + xdsClient.watchEdsResource(EDS_RESOURCE, edsResourceWatcher); + RpcCall call = resourceDiscoveryCalls.poll(); + List clusters = + ImmutableList.of(Any.pack(buildCluster(CDS_RESOURCE, EDS_RESOURCE, false))); + DiscoveryResponse response = + buildDiscoveryResponse("0", clusters, ResourceType.CDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + verify(cdsResourceWatcher).onChanged(cdsUpdateCaptor.capture()); + assertThat(cdsUpdateCaptor.getValue().getClusterName()).isEqualTo(CDS_RESOURCE); + assertThat(cdsUpdateCaptor.getValue().getEdsServiceName()).isEqualTo(EDS_RESOURCE); + + List clusterLoadAssignments = + ImmutableList.of( + Any.pack( + buildClusterLoadAssignment(EDS_RESOURCE, + ImmutableList.of( + buildLocalityLbEndpoints("region1", "zone1", "subzone1", + ImmutableList.of( + buildLbEndpoint("192.168.0.1", 8080, HealthStatus.HEALTHY, 2)), + 1, 0)), + ImmutableList.of( + buildDropOverload("lb", 200), + buildDropOverload("throttle", 1000))))); + response = + buildDiscoveryResponse("0", clusterLoadAssignments, ResourceType.EDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + verify(edsResourceWatcher).onChanged(edsUpdateCaptor.capture()); + EdsUpdate edsUpdate = edsUpdateCaptor.getValue(); + assertThat(edsUpdate.getClusterName()).isEqualTo(EDS_RESOURCE); + + clusters = ImmutableList.of(Any.pack(buildCluster(CDS_RESOURCE, null, false))); + response = + buildDiscoveryResponse("1", clusters, ResourceType.CDS.typeUrl(), "0001"); + call.responseObserver.onNext(response); + verify(cdsResourceWatcher, times(2)).onChanged(cdsUpdateCaptor.capture()); + assertThat(cdsUpdateCaptor.getValue().getClusterName()).isEqualTo(CDS_RESOURCE); + assertThat(cdsUpdateCaptor.getValue().getEdsServiceName()).isNull(); + verify(edsResourceWatcher).onResourceDoesNotExist(EDS_RESOURCE); + } + + @Test + public void multipleEdsWatchers() { + String edsResource = "cluster-load-assignment-bar.googleapis.com"; + EdsResourceWatcher watcher1 = mock(EdsResourceWatcher.class); + EdsResourceWatcher watcher2 = mock(EdsResourceWatcher.class); + xdsClient.watchEdsResource(EDS_RESOURCE, edsResourceWatcher); + xdsClient.watchEdsResource(edsResource, watcher1); + xdsClient.watchEdsResource(edsResource, watcher2); + RpcCall call = resourceDiscoveryCalls.poll(); + verify(call.requestObserver).onNext( + argThat(new DiscoveryRequestMatcher(NODE, "", Arrays.asList(EDS_RESOURCE, edsResource), + ResourceType.EDS.typeUrl(), ""))); + + fakeClock.forwardTime(XdsClientImpl2.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); + verify(edsResourceWatcher).onResourceDoesNotExist(EDS_RESOURCE); + verify(watcher1).onResourceDoesNotExist(edsResource); + verify(watcher2).onResourceDoesNotExist(edsResource); + + List clusterLoadAssignments = + ImmutableList.of( + Any.pack( + buildClusterLoadAssignment(EDS_RESOURCE, + ImmutableList.of( + buildLocalityLbEndpoints("region1", "zone1", "subzone1", + ImmutableList.of( + buildLbEndpoint("192.168.0.1", 8080, HealthStatus.HEALTHY, 2)), + 1, 0), + buildLocalityLbEndpoints("region3", "zone3", "subzone3", + ImmutableList.of(), + 2, 1), /* locality with 0 endpoint */ + buildLocalityLbEndpoints("region4", "zone4", "subzone4", + ImmutableList.of( + buildLbEndpoint("192.168.142.5", 80, HealthStatus.UNKNOWN, 5)), + 0, 2) /* locality with 0 weight */), + ImmutableList.of( + buildDropOverload("lb", 200), + buildDropOverload("throttle", 1000))))); + DiscoveryResponse response = + buildDiscoveryResponse("0", clusterLoadAssignments, ResourceType.EDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + verify(edsResourceWatcher).onChanged(edsUpdateCaptor.capture()); + EdsUpdate edsUpdate = edsUpdateCaptor.getValue(); + assertThat(edsUpdate.getClusterName()).isEqualTo(EDS_RESOURCE); + assertThat(edsUpdate.getDropPolicies()) + .containsExactly( + new DropOverload("lb", 200), + new DropOverload("throttle", 1000)); + assertThat(edsUpdate.getLocalityLbEndpointsMap()) + .containsExactly( + new Locality("region1", "zone1", "subzone1"), + new LocalityLbEndpoints( + ImmutableList.of( + new LbEndpoint("192.168.0.1", 8080, 2, true)), 1, 0), + new Locality("region3", "zone3", "subzone3"), + new LocalityLbEndpoints(ImmutableList.of(), 2, 1)); + verifyNoMoreInteractions(watcher1, watcher2); + + clusterLoadAssignments = + ImmutableList.of( + Any.pack( + buildClusterLoadAssignment(edsResource, + ImmutableList.of( + buildLocalityLbEndpoints("region2", "zone2", "subzone2", + ImmutableList.of( + buildLbEndpoint("172.44.2.2", 8000, HealthStatus.HEALTHY, 3)), + 2, 0)), + ImmutableList.of()))); + response = + buildDiscoveryResponse("1", clusterLoadAssignments, ResourceType.EDS.typeUrl(), "0001"); + call.responseObserver.onNext(response); + + verify(watcher1).onChanged(edsUpdateCaptor.capture()); + edsUpdate = edsUpdateCaptor.getValue(); + assertThat(edsUpdate.getClusterName()).isEqualTo(edsResource); + assertThat(edsUpdate.getDropPolicies()).isEmpty(); + assertThat(edsUpdate.getLocalityLbEndpointsMap()) + .containsExactly( + new Locality("region2", "zone2", "subzone2"), + new LocalityLbEndpoints( + ImmutableList.of( + new LbEndpoint("172.44.2.2", 8000, 3, true)), 2, 0)); + verify(watcher2).onChanged(edsUpdateCaptor.capture()); + edsUpdate = edsUpdateCaptor.getValue(); + assertThat(edsUpdate.getClusterName()).isEqualTo(edsResource); + assertThat(edsUpdate.getDropPolicies()).isEmpty(); + assertThat(edsUpdate.getLocalityLbEndpointsMap()) + .containsExactly( + new Locality("region2", "zone2", "subzone2"), + new LocalityLbEndpoints( + ImmutableList.of( + new LbEndpoint("172.44.2.2", 8000, 3, true)), 2, 0)); + verifyNoMoreInteractions(edsResourceWatcher); + } + + @Test + public void streamClosedAndRetryWithBackoff() { + InOrder inOrder = Mockito.inOrder(backoffPolicyProvider, backoffPolicy1, backoffPolicy2); + xdsClient.watchLdsResource(LDS_RESOURCE, ldsResourceWatcher); + xdsClient.watchRdsResource(RDS_RESOURCE, rdsResourceWatcher); + xdsClient.watchCdsResource(CDS_RESOURCE, cdsResourceWatcher); + xdsClient.watchEdsResource(EDS_RESOURCE, edsResourceWatcher); + RpcCall call = resourceDiscoveryCalls.poll(); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "", LDS_RESOURCE, ResourceType.LDS.typeUrl(), ""))); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "", RDS_RESOURCE, ResourceType.RDS.typeUrl(), ""))); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "", CDS_RESOURCE, ResourceType.CDS.typeUrl(), ""))); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "", EDS_RESOURCE, ResourceType.EDS.typeUrl(), ""))); + + // Management server closes the RPC stream with an error. + call.responseObserver.onError(Status.UNKNOWN.asException()); + verify(ldsResourceWatcher).onError(errorCaptor.capture()); + assertThat(errorCaptor.getValue().getCode()).isEqualTo(Code.UNKNOWN); + verify(rdsResourceWatcher).onError(errorCaptor.capture()); + assertThat(errorCaptor.getValue().getCode()).isEqualTo(Code.UNKNOWN); + verify(cdsResourceWatcher).onError(errorCaptor.capture()); + assertThat(errorCaptor.getValue().getCode()).isEqualTo(Code.UNKNOWN); + verify(edsResourceWatcher).onError(errorCaptor.capture()); + assertThat(errorCaptor.getValue().getCode()).isEqualTo(Code.UNKNOWN); + + // Retry after backoff. + inOrder.verify(backoffPolicyProvider).get(); + inOrder.verify(backoffPolicy1).nextBackoffNanos(); + ScheduledTask retryTask = + Iterables.getOnlyElement(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)); + assertThat(retryTask.getDelay(TimeUnit.NANOSECONDS)).isEqualTo(10L); + fakeClock.forwardNanos(10L); + call = resourceDiscoveryCalls.poll(); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "", LDS_RESOURCE, ResourceType.LDS.typeUrl(), ""))); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "", RDS_RESOURCE, ResourceType.RDS.typeUrl(), ""))); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "", CDS_RESOURCE, ResourceType.CDS.typeUrl(), ""))); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "", EDS_RESOURCE, ResourceType.EDS.typeUrl(), ""))); + + // Management server becomes unreachable. + call.responseObserver.onError(Status.UNAVAILABLE.asException()); + verify(ldsResourceWatcher, times(2)).onError(errorCaptor.capture()); + assertThat(errorCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); + verify(rdsResourceWatcher, times(2)).onError(errorCaptor.capture()); + assertThat(errorCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); + verify(cdsResourceWatcher, times(2)).onError(errorCaptor.capture()); + assertThat(errorCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); + verify(edsResourceWatcher, times(2)).onError(errorCaptor.capture()); + assertThat(errorCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); + + // Retry after backoff. + inOrder.verify(backoffPolicy1).nextBackoffNanos(); + retryTask = + Iterables.getOnlyElement(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)); + assertThat(retryTask.getDelay(TimeUnit.NANOSECONDS)).isEqualTo(100L); + fakeClock.forwardNanos(100L); + call = resourceDiscoveryCalls.poll(); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "", LDS_RESOURCE, ResourceType.LDS.typeUrl(), ""))); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "", RDS_RESOURCE, ResourceType.RDS.typeUrl(), ""))); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "", CDS_RESOURCE, ResourceType.CDS.typeUrl(), ""))); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "", EDS_RESOURCE, ResourceType.EDS.typeUrl(), ""))); + + List listeners = ImmutableList.of( + Any.pack(buildListener(LDS_RESOURCE, + Any.pack( + HttpConnectionManager.newBuilder() + .setRouteConfig(buildRouteConfiguration("do not care", buildVirtualHosts(2))) + .build())))); + DiscoveryResponse response = + buildDiscoveryResponse("63", listeners, ResourceType.LDS.typeUrl(), "3242"); + call.responseObserver.onNext(response); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "63", LDS_RESOURCE, ResourceType.LDS.typeUrl(), "3242"))); + + List routeConfigs = + ImmutableList.of(Any.pack(buildRouteConfiguration(RDS_RESOURCE, buildVirtualHosts(2)))); + response = + buildDiscoveryResponse("5", routeConfigs, ResourceType.RDS.typeUrl(), "6764"); + call.responseObserver.onNext(response); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "5", RDS_RESOURCE, ResourceType.RDS.typeUrl(), "6764"))); + + call.responseObserver.onError(Status.DEADLINE_EXCEEDED.asException()); + verify(ldsResourceWatcher, times(3)).onError(errorCaptor.capture()); + assertThat(errorCaptor.getValue().getCode()).isEqualTo(Code.DEADLINE_EXCEEDED); + verify(rdsResourceWatcher, times(3)).onError(errorCaptor.capture()); + assertThat(errorCaptor.getValue().getCode()).isEqualTo(Code.DEADLINE_EXCEEDED); + verify(cdsResourceWatcher, times(3)).onError(errorCaptor.capture()); + assertThat(errorCaptor.getValue().getCode()).isEqualTo(Code.DEADLINE_EXCEEDED); + verify(edsResourceWatcher, times(3)).onError(errorCaptor.capture()); + assertThat(errorCaptor.getValue().getCode()).isEqualTo(Code.DEADLINE_EXCEEDED); + + // Reset backoff sequence and retry immediately. + inOrder.verify(backoffPolicyProvider).get(); + fakeClock.runDueTasks(); + call = resourceDiscoveryCalls.poll(); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "63", LDS_RESOURCE, ResourceType.LDS.typeUrl(), ""))); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "5", RDS_RESOURCE, ResourceType.RDS.typeUrl(), ""))); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "", CDS_RESOURCE, ResourceType.CDS.typeUrl(), ""))); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "", EDS_RESOURCE, ResourceType.EDS.typeUrl(), ""))); + + // Management server becomes unreachable again. + call.responseObserver.onError(Status.UNAVAILABLE.asException()); + verify(ldsResourceWatcher, times(4)).onError(errorCaptor.capture()); + assertThat(errorCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); + verify(rdsResourceWatcher, times(4)).onError(errorCaptor.capture()); + assertThat(errorCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); + verify(cdsResourceWatcher, times(4)).onError(errorCaptor.capture()); + assertThat(errorCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); + verify(edsResourceWatcher, times(4)).onError(errorCaptor.capture()); + assertThat(errorCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); + + // Retry after backoff. + inOrder.verify(backoffPolicy2).nextBackoffNanos(); + retryTask = + Iterables.getOnlyElement(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)); + assertThat(retryTask.getDelay(TimeUnit.NANOSECONDS)).isEqualTo(20L); + fakeClock.forwardNanos(20L); + call = resourceDiscoveryCalls.poll(); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "63", LDS_RESOURCE, ResourceType.LDS.typeUrl(), ""))); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "5", RDS_RESOURCE, ResourceType.RDS.typeUrl(), ""))); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "", CDS_RESOURCE, ResourceType.CDS.typeUrl(), ""))); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "", EDS_RESOURCE, ResourceType.EDS.typeUrl(), ""))); + + inOrder.verifyNoMoreInteractions(); + } + + @Test + public void streamClosedAndRetryRaceWithAddRemoveWatchers() { + xdsClient.watchLdsResource(LDS_RESOURCE, ldsResourceWatcher); + xdsClient.watchRdsResource(RDS_RESOURCE, rdsResourceWatcher); + RpcCall call = resourceDiscoveryCalls.poll(); + call.responseObserver.onError(Status.UNAVAILABLE.asException()); + verify(ldsResourceWatcher).onError(errorCaptor.capture()); + assertThat(errorCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); + verify(rdsResourceWatcher).onError(errorCaptor.capture()); + assertThat(errorCaptor.getValue().getCode()).isEqualTo(Code.UNAVAILABLE); + ScheduledTask retryTask = + Iterables.getOnlyElement(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)); + assertThat(retryTask.getDelay(TimeUnit.NANOSECONDS)).isEqualTo(10L); + + xdsClient.cancelLdsResourceWatch(LDS_RESOURCE, ldsResourceWatcher); + xdsClient.cancelRdsResourceWatch(RDS_RESOURCE, rdsResourceWatcher); + xdsClient.watchCdsResource(CDS_RESOURCE, cdsResourceWatcher); + xdsClient.watchEdsResource(EDS_RESOURCE, edsResourceWatcher); + fakeClock.forwardNanos(10L); + call = resourceDiscoveryCalls.poll(); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "", CDS_RESOURCE, ResourceType.CDS.typeUrl(), ""))); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "", EDS_RESOURCE, ResourceType.EDS.typeUrl(), ""))); + verifyNoMoreInteractions(call.requestObserver); + + List listeners = ImmutableList.of( + Any.pack(buildListener(LDS_RESOURCE, + Any.pack( + HttpConnectionManager.newBuilder() + .setRouteConfig(buildRouteConfiguration("do not care", buildVirtualHosts(2))) + .build())))); + DiscoveryResponse response = + buildDiscoveryResponse("0", listeners, ResourceType.LDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + List routeConfigs = + ImmutableList.of(Any.pack(buildRouteConfiguration(RDS_RESOURCE, buildVirtualHosts(2)))); + response = + buildDiscoveryResponse("0", routeConfigs, ResourceType.RDS.typeUrl(), "0000"); + call.responseObserver.onNext(response); + + verifyNoMoreInteractions(ldsResourceWatcher, rdsResourceWatcher); + } + + @Test + public void streamClosedAndRetryRestartResourceInitialFetchTimers() { + xdsClient.watchLdsResource(LDS_RESOURCE, ldsResourceWatcher); + xdsClient.watchRdsResource(RDS_RESOURCE, rdsResourceWatcher); + xdsClient.watchCdsResource(CDS_RESOURCE, cdsResourceWatcher); + xdsClient.watchEdsResource(EDS_RESOURCE, edsResourceWatcher); + RpcCall call = resourceDiscoveryCalls.poll(); + ScheduledTask ldsResourceTimeout = + Iterables.getOnlyElement(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); + ScheduledTask rdsResourceTimeout = + Iterables.getOnlyElement(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); + ScheduledTask cdsResourceTimeout = + Iterables.getOnlyElement(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); + ScheduledTask edsResourceTimeout = + Iterables.getOnlyElement(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); + call.responseObserver.onError(Status.UNAVAILABLE.asException()); + assertThat(ldsResourceTimeout.isCancelled()).isTrue(); + assertThat(rdsResourceTimeout.isCancelled()).isTrue(); + assertThat(cdsResourceTimeout.isCancelled()).isTrue(); + assertThat(edsResourceTimeout.isCancelled()).isTrue(); + + fakeClock.forwardNanos(10L); + assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + } + + /** + * Tests sending a streaming LRS RPC for each cluster to report loads for. + */ + @Test + public void reportLoadStatsToServer() { + String clusterName = "cluster-foo.googleapis.com"; + xdsClient.addClientStats(clusterName, null); + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(null); + xdsClient.reportClientStats(); + RpcCall lrsCall = loadReportCalls.poll(); + verify(lrsCall.requestObserver).onNext(requestCaptor.capture()); + assertThat(requestCaptor.getValue().getClusterStatsCount()) + .isEqualTo(0); // initial request + + lrsCall.responseObserver.onNext( + LoadStatsResponse.newBuilder() + .addClusters(clusterName) + .setLoadReportingInterval(Durations.fromNanos(1000L)) + .build()); + fakeClock.forwardNanos(1000L); + verify(lrsCall.requestObserver, times(2)).onNext(requestCaptor.capture()); + ClusterStats report = Iterables.getOnlyElement(requestCaptor.getValue().getClusterStatsList()); + assertThat(report.getClusterName()).isEqualTo(clusterName); + + xdsClient.removeClientStats(clusterName, null); + fakeClock.forwardNanos(1000L); + verify(lrsCall.requestObserver, times(3)).onNext(requestCaptor.capture()); + assertThat(requestCaptor.getValue().getClusterStatsCount()) + .isEqualTo(0); // no more stats reported + + xdsClient.cancelClientStatsReport(); + assertThat(lrsEnded.get()).isTrue(); + // See more test on LoadReportClientTest.java + } + + @Test + public void messagePrinter_printLdsResponse() { + MessagePrinter printer = new MessagePrinter(); + List listeners = ImmutableList.of( + Any.pack(buildListener("foo.googleapis.com:8080", + Any.pack( + HttpConnectionManager.newBuilder() + .setRouteConfig( + buildRouteConfiguration("route-foo.googleapis.com", + ImmutableList.of( + buildVirtualHost( + ImmutableList.of("foo.googleapis.com", "bar.googleapis.com"), + "cluster.googleapis.com")))) + .build())))); + DiscoveryResponse response = + buildDiscoveryResponse("0", listeners, ResourceType.LDS.typeUrl(), "0000"); + + String expectedString = "{\n" + + " \"versionInfo\": \"0\",\n" + + " \"resources\": [{\n" + + " \"@type\": \"type.googleapis.com/envoy.config.listener.v3.Listener\",\n" + + " \"name\": \"foo.googleapis.com:8080\",\n" + + " \"address\": {\n" + + " },\n" + + " \"filterChains\": [{\n" + + " }],\n" + + " \"apiListener\": {\n" + + " \"apiListener\": {\n" + + " \"@type\": \"type.googleapis.com/envoy.extensions.filters.network" + + ".http_connection_manager.v3.HttpConnectionManager\",\n" + + " \"routeConfig\": {\n" + + " \"name\": \"route-foo.googleapis.com\",\n" + + " \"virtualHosts\": [{\n" + + " \"name\": \"virtualhost00.googleapis.com\",\n" + + " \"domains\": [\"foo.googleapis.com\", \"bar.googleapis.com\"],\n" + + " \"routes\": [{\n" + + " \"match\": {\n" + + " \"prefix\": \"\"\n" + + " },\n" + + " \"route\": {\n" + + " \"cluster\": \"cluster.googleapis.com\"\n" + + " }\n" + + " }]\n" + + " }]\n" + + " }\n" + + " }\n" + + " }\n" + + " }],\n" + + " \"typeUrl\": \"type.googleapis.com/envoy.config.listener.v3.Listener\",\n" + + " \"nonce\": \"0000\"\n" + + "}"; + String res = printer.print(response); + assertThat(res).isEqualTo(expectedString); + } + + @Test + public void messagePrinter_printRdsResponse() { + MessagePrinter printer = new MessagePrinter(); + List routeConfigs = + ImmutableList.of( + Any.pack( + buildRouteConfiguration( + "route-foo.googleapis.com", + ImmutableList.of( + buildVirtualHost( + ImmutableList.of("foo.googleapis.com", "bar.googleapis.com"), + "cluster.googleapis.com"))))); + DiscoveryResponse response = + buildDiscoveryResponse("213", routeConfigs, ResourceType.RDS.typeUrl(), "0052"); + + String expectedString = "{\n" + + " \"versionInfo\": \"213\",\n" + + " \"resources\": [{\n" + + " \"@type\": \"type.googleapis.com/envoy.config.route.v3.RouteConfiguration\",\n" + + " \"name\": \"route-foo.googleapis.com\",\n" + + " \"virtualHosts\": [{\n" + + " \"name\": \"virtualhost00.googleapis.com\",\n" + + " \"domains\": [\"foo.googleapis.com\", \"bar.googleapis.com\"],\n" + + " \"routes\": [{\n" + + " \"match\": {\n" + + " \"prefix\": \"\"\n" + + " },\n" + + " \"route\": {\n" + + " \"cluster\": \"cluster.googleapis.com\"\n" + + " }\n" + + " }]\n" + + " }]\n" + + " }],\n" + + " \"typeUrl\": \"type.googleapis.com/envoy.config.route.v3.RouteConfiguration\",\n" + + " \"nonce\": \"0052\"\n" + + "}"; + String res = printer.print(response); + assertThat(res).isEqualTo(expectedString); + } + + @Test + public void messagePrinter_printCdsResponse() { + MessagePrinter printer = new MessagePrinter(); + List clusters = ImmutableList.of( + Any.pack(buildCluster("cluster-bar.googleapis.com", "service-blaze:cluster-bar", true)), + Any.pack(buildCluster("cluster-foo.googleapis.com", null, false))); + DiscoveryResponse response = + buildDiscoveryResponse("14", clusters, ResourceType.CDS.typeUrl(), "8"); + + String expectedString = "{\n" + + " \"versionInfo\": \"14\",\n" + + " \"resources\": [{\n" + + " \"@type\": \"type.googleapis.com/envoy.config.cluster.v3.Cluster\",\n" + + " \"name\": \"cluster-bar.googleapis.com\",\n" + + " \"type\": \"EDS\",\n" + + " \"edsClusterConfig\": {\n" + + " \"edsConfig\": {\n" + + " \"ads\": {\n" + + " }\n" + + " },\n" + + " \"serviceName\": \"service-blaze:cluster-bar\"\n" + + " },\n" + + " \"lrsServer\": {\n" + + " \"self\": {\n" + + " }\n" + + " }\n" + + " }, {\n" + + " \"@type\": \"type.googleapis.com/envoy.config.cluster.v3.Cluster\",\n" + + " \"name\": \"cluster-foo.googleapis.com\",\n" + + " \"type\": \"EDS\",\n" + + " \"edsClusterConfig\": {\n" + + " \"edsConfig\": {\n" + + " \"ads\": {\n" + + " }\n" + + " }\n" + + " }\n" + + " }],\n" + + " \"typeUrl\": \"type.googleapis.com/envoy.config.cluster.v3.Cluster\",\n" + + " \"nonce\": \"8\"\n" + + "}"; + String res = printer.print(response); + assertThat(res).isEqualTo(expectedString); + } + + @Test + public void messagePrinter_printEdsResponse() { + MessagePrinter printer = new MessagePrinter(); + List clusterLoadAssignments = ImmutableList.of( + Any.pack(buildClusterLoadAssignment("cluster-foo.googleapis.com", + ImmutableList.of( + buildLocalityLbEndpoints("region1", "zone1", "subzone1", + ImmutableList.of( + buildLbEndpoint("192.168.0.1", 8080, HealthStatus.HEALTHY, 2)), + 1, 0), + buildLocalityLbEndpoints("region3", "zone3", "subzone3", + ImmutableList.of( + buildLbEndpoint("192.168.142.5", 80, HealthStatus.UNHEALTHY, 5)), + 2, 1)), + ImmutableList.of( + buildDropOverload("lb", 200), + buildDropOverload("throttle", 1000))))); + + DiscoveryResponse response = + buildDiscoveryResponse("5", clusterLoadAssignments, + ResourceType.EDS.typeUrl(), "004"); + + String expectedString = "{\n" + + " \"versionInfo\": \"5\",\n" + + " \"resources\": [{\n" + + " \"@type\": \"type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment\",\n" + + " \"clusterName\": \"cluster-foo.googleapis.com\",\n" + + " \"endpoints\": [{\n" + + " \"locality\": {\n" + + " \"region\": \"region1\",\n" + + " \"zone\": \"zone1\",\n" + + " \"subZone\": \"subzone1\"\n" + + " },\n" + + " \"lbEndpoints\": [{\n" + + " \"endpoint\": {\n" + + " \"address\": {\n" + + " \"socketAddress\": {\n" + + " \"address\": \"192.168.0.1\",\n" + + " \"portValue\": 8080\n" + + " }\n" + + " }\n" + + " },\n" + + " \"healthStatus\": \"HEALTHY\",\n" + + " \"loadBalancingWeight\": 2\n" + + " }],\n" + + " \"loadBalancingWeight\": 1\n" + + " }, {\n" + + " \"locality\": {\n" + + " \"region\": \"region3\",\n" + + " \"zone\": \"zone3\",\n" + + " \"subZone\": \"subzone3\"\n" + + " },\n" + + " \"lbEndpoints\": [{\n" + + " \"endpoint\": {\n" + + " \"address\": {\n" + + " \"socketAddress\": {\n" + + " \"address\": \"192.168.142.5\",\n" + + " \"portValue\": 80\n" + + " }\n" + + " }\n" + + " },\n" + + " \"healthStatus\": \"UNHEALTHY\",\n" + + " \"loadBalancingWeight\": 5\n" + + " }],\n" + + " \"loadBalancingWeight\": 2,\n" + + " \"priority\": 1\n" + + " }],\n" + + " \"policy\": {\n" + + " \"dropOverloads\": [{\n" + + " \"category\": \"lb\",\n" + + " \"dropPercentage\": {\n" + + " \"numerator\": 200,\n" + + " \"denominator\": \"MILLION\"\n" + + " }\n" + + " }, {\n" + + " \"category\": \"throttle\",\n" + + " \"dropPercentage\": {\n" + + " \"numerator\": 1000,\n" + + " \"denominator\": \"MILLION\"\n" + + " }\n" + + " }]\n" + + " }\n" + + " }],\n" + + " \"typeUrl\": \"type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment\",\n" + + " \"nonce\": \"004\"\n" + + "}"; + String res = printer.print(response); + assertThat(res).isEqualTo(expectedString); + } + + private RpcCall startResourceWatcher( + ResourceType type, String name, ResourceWatcher watcher) { + FakeClock.TaskFilter timeoutTaskFilter; + switch (type) { + case LDS: + timeoutTaskFilter = LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER; + xdsClient.watchLdsResource(name, (LdsResourceWatcher) watcher); + break; + case RDS: + timeoutTaskFilter = RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER; + xdsClient.watchRdsResource(name, (RdsResourceWatcher) watcher); + break; + case CDS: + timeoutTaskFilter = CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER; + xdsClient.watchCdsResource(name, (CdsResourceWatcher) watcher); + break; + case EDS: + timeoutTaskFilter = EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER; + xdsClient.watchEdsResource(name, (EdsResourceWatcher) watcher); + break; + case UNKNOWN: + default: + throw new AssertionError("should never be here"); + } + RpcCall call = resourceDiscoveryCalls.poll(); + verify(call.requestObserver).onNext( + eq(buildDiscoveryRequest(NODE, "", name, type.typeUrl(), ""))); + ScheduledTask timeoutTask = + Iterables.getOnlyElement(fakeClock.getPendingTasks(timeoutTaskFilter)); + assertThat(timeoutTask.getDelay(TimeUnit.SECONDS)) + .isEqualTo(XdsClientImpl2.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC); + return call; + } + + /** + * Matcher for DiscoveryRequest without the comparison of error_details field, which is used for + * management server debugging purposes. + * + *

In general, if you are sure error_details field should not be set in a DiscoveryRequest, + * compare with message equality. Otherwise, this matcher is handy for comparing other fields + * only. + */ + private static class DiscoveryRequestMatcher implements ArgumentMatcher { + private final Node node; + private final String versionInfo; + private final String typeUrl; + private final Set resourceNames; + private final String responseNonce; + + private DiscoveryRequestMatcher( + Node node, String versionInfo, List resourceNames, String typeUrl, + String responseNonce) { + this.node = node; + this.versionInfo = versionInfo; + this.resourceNames = new HashSet<>(resourceNames); + this.typeUrl = typeUrl; + this.responseNonce = responseNonce; + } + + @Override + public boolean matches(DiscoveryRequest argument) { + if (!typeUrl.equals(argument.getTypeUrl())) { + return false; + } + if (!versionInfo.equals(argument.getVersionInfo())) { + return false; + } + if (!responseNonce.equals(argument.getResponseNonce())) { + return false; + } + if (!node.toEnvoyProtoNode().equals(argument.getNode())) { + return false; + } + if (!resourceNames.equals(new HashSet<>(argument.getResourceNamesList()))) { + return false; + } + return argument.getNode().equals(NODE.toEnvoyProtoNode()); + } + } + + private static class RpcCall { + private final StreamObserver requestObserver; + private final StreamObserver responseObserver; + + RpcCall(StreamObserver requestObserver, StreamObserver responseObserver) { + this.requestObserver = requestObserver; + this.responseObserver = responseObserver; + } + } +} diff --git a/xds/src/test/java/io/grpc/xds/XdsClientTestHelper.java b/xds/src/test/java/io/grpc/xds/XdsClientTestHelper.java index 3220a77e524..08ba88aad26 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientTestHelper.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientTestHelper.java @@ -56,6 +56,7 @@ import io.envoyproxy.envoy.type.v3.FractionalPercent; import io.envoyproxy.envoy.type.v3.FractionalPercent.DenominatorType; import io.grpc.xds.EnvoyProtoData.Node; +import java.util.ArrayList; import java.util.List; import javax.annotation.Nullable; @@ -160,6 +161,23 @@ static io.envoyproxy.envoy.api.v2.RouteConfiguration buildRouteConfigurationV2(S .build(); } + static List buildVirtualHosts(int num) { + List virtualHosts = new ArrayList<>(num); + for (int i = 0; i < num; i++) { + VirtualHost virtualHost = + VirtualHost.newBuilder() + .setName(num + ": do not care") + .addDomains("do not care") + .addRoutes( + Route.newBuilder() + .setRoute(RouteAction.newBuilder().setCluster("do not care")) + .setMatch(RouteMatch.newBuilder().setPrefix("do not care"))) + .build(); + virtualHosts.add(virtualHost); + } + return virtualHosts; + } + static VirtualHost buildVirtualHost(List domains, String clusterName) { return VirtualHost.newBuilder() .setName("virtualhost00.googleapis.com") // don't care From cdeda507586e546d7195a34b789b530a614cbd9b Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Mon, 5 Oct 2020 09:51:27 -0700 Subject: [PATCH 81/86] netty: Fix Javadoc for ShadingTest --- .../src/testShadow/java/io/grpc/netty/shaded/ShadingTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netty/shaded/src/testShadow/java/io/grpc/netty/shaded/ShadingTest.java b/netty/shaded/src/testShadow/java/io/grpc/netty/shaded/ShadingTest.java index 8527ff4ebdd..20f6fa1b6f2 100644 --- a/netty/shaded/src/testShadow/java/io/grpc/netty/shaded/ShadingTest.java +++ b/netty/shaded/src/testShadow/java/io/grpc/netty/shaded/ShadingTest.java @@ -40,7 +40,7 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -/** Unit tests for {@link Shading}. */ +/** Unit tests for shaded gRPC Netty. */ @RunWith(JUnit4.class) public final class ShadingTest { private ManagedChannel channel; From 0913dd27692e33269cfb2fb76189fb52e47a2fe4 Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Tue, 6 Oct 2020 13:51:33 -0700 Subject: [PATCH 82/86] xds: fix lint (#7487) --- .../main/java/io/grpc/xds/XdsClientImpl2.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/XdsClientImpl2.java b/xds/src/main/java/io/grpc/xds/XdsClientImpl2.java index d4c7e5f58d4..02c943721b1 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientImpl2.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientImpl2.java @@ -27,6 +27,7 @@ import com.google.protobuf.Any; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.MessageOrBuilder; +import com.google.protobuf.TypeRegistry; import com.google.protobuf.util.Durations; import com.google.protobuf.util.JsonFormat; import com.google.rpc.Code; @@ -45,6 +46,7 @@ import io.envoyproxy.envoy.config.route.v3.VirtualHost; import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager; import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.Rds; +import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext; import io.envoyproxy.envoy.service.discovery.v3.AggregatedDiscoveryServiceGrpc; import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest; import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse; @@ -59,7 +61,6 @@ import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints; import io.grpc.xds.EnvoyProtoData.Node; import io.grpc.xds.EnvoyProtoData.StructOrError; -import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import io.grpc.xds.LoadStatsManager.LoadStatsStore; import io.grpc.xds.XdsLogger.XdsLogLevel; import java.util.ArrayList; @@ -745,7 +746,8 @@ private void handleCdsResponse(DiscoveryResponseData cdsResponse) { updateBuilder.setLrsServerName(""); } try { - UpstreamTlsContext upstreamTlsContext = getTlsContextFromCluster(cluster); + EnvoyServerProtoData.UpstreamTlsContext upstreamTlsContext = + getTlsContextFromCluster(cluster); if (upstreamTlsContext != null && upstreamTlsContext.getCommonTlsContext() != null) { updateBuilder.setUpstreamTlsContext(upstreamTlsContext); } @@ -783,13 +785,12 @@ private void handleCdsResponse(DiscoveryResponseData cdsResponse) { } @Nullable - private static UpstreamTlsContext getTlsContextFromCluster(Cluster cluster) + private static EnvoyServerProtoData.UpstreamTlsContext getTlsContextFromCluster(Cluster cluster) throws InvalidProtocolBufferException { if (cluster.hasTransportSocket() && "tls".equals(cluster.getTransportSocket().getName())) { Any any = cluster.getTransportSocket().getTypedConfig(); - return UpstreamTlsContext.fromEnvoyProtoUpstreamTlsContext( - io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext.parseFrom( - any.getValue())); + return EnvoyServerProtoData.UpstreamTlsContext.fromEnvoyProtoUpstreamTlsContext( + any.unpack(UpstreamTlsContext.class)); } return null; } @@ -1670,8 +1671,8 @@ static final class MessagePrinter { @VisibleForTesting MessagePrinter() { - com.google.protobuf.TypeRegistry registry = - com.google.protobuf.TypeRegistry.newBuilder() + TypeRegistry registry = + TypeRegistry.newBuilder() .add(Listener.getDescriptor()) .add(io.envoyproxy.envoy.api.v2.Listener.getDescriptor()) .add(HttpConnectionManager.getDescriptor()) From 0e0bcdfe2a275fd3eea1e1617aa9c4eaab5844a4 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Tue, 6 Oct 2020 13:15:16 -0700 Subject: [PATCH 83/86] repositories.bzl: Remove Maven repositories, in favor of maven_install Manually specifying individual Maven artifacts is very verbose and error-prone. It also does not properly handle transitive dependencies. It greatly increases the amount of effort to update dependencies. v1.27.0 introduced support for maven_install and encouraged users to migrate. I fully expect some users haven't migrated, but it's not clear that an additional 8 months would help much. Users that don't want to use maven_install are still free to do so, but would need to maintain the verbose repository list themselves. At some point we may begin using the @maven workspace which would require maven_install, but are not doing so now (except in the examples) and don't have immediate plans to start. --- examples/WORKSPACE | 2 - repositories.bzl | 354 --------------------------------------------- 2 files changed, 356 deletions(-) diff --git a/examples/WORKSPACE b/examples/WORKSPACE index 1e5bad7f9f9..36a002b4ee0 100644 --- a/examples/WORKSPACE +++ b/examples/WORKSPACE @@ -43,8 +43,6 @@ compat_repositories() load("@io_grpc_grpc_java//:repositories.bzl", "grpc_java_repositories") -# Run grpc_java_repositories after compat_repositories to ensure the -# maven_install-selected dependencies are used. grpc_java_repositories() load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") diff --git a/repositories.bzl b/repositories.bzl index 9c4ab886bac..1fa113f8e64 100644 --- a/repositories.bzl +++ b/repositories.bzl @@ -92,71 +92,6 @@ def grpc_java_repositories(): if not native.existing_rule("io_grpc_grpc_proto"): io_grpc_grpc_proto() - if not native.existing_rule("com_google_android_annotations"): - com_google_android_annotations() - if not native.existing_rule("com_google_api_grpc_proto_google_common_protos"): - com_google_api_grpc_proto_google_common_protos() - if not native.existing_rule("com_google_auth_google_auth_library_credentials"): - com_google_auth_google_auth_library_credentials() - if not native.existing_rule("com_google_auth_google_auth_library_oauth2_http"): - com_google_auth_google_auth_library_oauth2_http() - if not native.existing_rule("com_google_code_findbugs_jsr305"): - com_google_code_findbugs_jsr305() - if not native.existing_rule("com_google_code_gson_gson"): - com_google_code_gson_gson() - if not native.existing_rule("com_google_errorprone_error_prone_annotations"): - com_google_errorprone_error_prone_annotations() - if not native.existing_rule("com_google_guava_guava"): - com_google_guava_guava() - if not native.existing_rule("com_google_guava_failureaccess"): - com_google_guava_failureaccess() - if not native.existing_rule("com_google_j2objc_j2objc_annotations"): - com_google_j2objc_j2objc_annotations() - if not native.existing_rule("com_google_truth_truth"): - com_google_truth_truth() - if not native.existing_rule("com_squareup_okhttp_okhttp"): - com_squareup_okhttp_okhttp() - if not native.existing_rule("com_squareup_okio_okio"): - com_squareup_okio_okio() - if not native.existing_rule("io_netty_netty_buffer"): - io_netty_netty_buffer() - if not native.existing_rule("io_netty_netty_codec"): - io_netty_netty_codec() - if not native.existing_rule("io_netty_netty_codec_http"): - io_netty_netty_codec_http() - if not native.existing_rule("io_netty_netty_codec_http2"): - io_netty_netty_codec_http2() - if not native.existing_rule("io_netty_netty_codec_socks"): - io_netty_netty_codec_socks() - if not native.existing_rule("io_netty_netty_common"): - io_netty_netty_common() - if not native.existing_rule("io_netty_netty_handler"): - io_netty_netty_handler() - if not native.existing_rule("io_netty_netty_handler_proxy"): - io_netty_netty_handler_proxy() - if not native.existing_rule("io_netty_netty_resolver"): - io_netty_netty_resolver() - if not native.existing_rule("io_netty_netty_tcnative_boringssl_static"): - io_netty_netty_tcnative_boringssl_static() - if not native.existing_rule("io_netty_netty_transport"): - io_netty_netty_transport() - if not native.existing_rule("io_netty_netty_transport_native_epoll_linux_x86_64"): - io_netty_netty_transport_native_epoll_linux_x86_64() - if not native.existing_rule("io_opencensus_opencensus_api"): - io_opencensus_opencensus_api() - if not native.existing_rule("io_opencensus_opencensus_contrib_grpc_metrics"): - io_opencensus_opencensus_contrib_grpc_metrics() - if not native.existing_rule("io_perfmark_perfmark_api"): - io_perfmark_perfmark_api() - if not native.existing_rule("junit_junit"): - junit_junit() - if not native.existing_rule("org_apache_commons_commons_lang3"): - org_apache_commons_commons_lang3() - if not native.existing_rule("org_apache_tomcat_annotations_api"): - org_apache_tomcat_annotations_api() - if not native.existing_rule("org_codehaus_mojo_animal_sniffer_annotations"): - org_codehaus_mojo_animal_sniffer_annotations() - native.bind( name = "guava", actual = "@com_google_guava_guava//jar", @@ -170,96 +105,6 @@ def grpc_java_repositories(): actual = "@com_google_errorprone_error_prone_annotations//jar", ) -def com_google_android_annotations(): - jvm_maven_import_external( - name = "com_google_android_annotations", - artifact = "com.google.android:annotations:4.1.1.4", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "ba734e1e84c09d615af6a09d33034b4f0442f8772dec120efb376d86a565ae15", - licenses = ["notice"], # Apache 2.0 - ) - -def com_google_api_grpc_proto_google_common_protos(): - jvm_maven_import_external( - name = "com_google_api_grpc_proto_google_common_protos", - artifact = "com.google.api.grpc:proto-google-common-protos:1.17.0", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "ad25472c73ee470606fb500b376ae5a97973d5406c2f5c3b7d07fb25b4648b65", - licenses = ["notice"], # Apache 2.0 - ) - -def com_google_auth_google_auth_library_credentials(): - jvm_maven_import_external( - name = "com_google_auth_google_auth_library_credentials", - artifact = "com.google.auth:google-auth-library-credentials:0.20.0", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "8a415273a5dae5c8f9080134e53b9592dc171ca5d13127488c910177c5903bd6", - licenses = ["notice"], # BSD 3-clause - ) - -def com_google_auth_google_auth_library_oauth2_http(): - jvm_maven_import_external( - name = "com_google_auth_google_auth_library_oauth2_http", - artifact = "com.google.auth:google-auth-library-oauth2-http:0.20.0", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "43e96e8c07285c2887042eda4e35ca96522ef361f6c1843f469039d9ccdc8f8a", - licenses = ["notice"], # BSD 3-clause - ) - -def com_google_code_findbugs_jsr305(): - jvm_maven_import_external( - name = "com_google_code_findbugs_jsr305", - artifact = "com.google.code.findbugs:jsr305:3.0.2", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "766ad2a0783f2687962c8ad74ceecc38a28b9f72a2d085ee438b7813e928d0c7", - licenses = ["notice"], # Apache 2.0 - ) - -def com_google_code_gson_gson(): - jvm_maven_import_external( - name = "com_google_code_gson_gson", - artifact = "com.google.code.gson:gson:jar:2.8.6", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "c8fb4839054d280b3033f800d1f5a97de2f028eb8ba2eb458ad287e536f3f25f", - licenses = ["notice"], # Apache 2.0 - ) - -def com_google_errorprone_error_prone_annotations(): - jvm_maven_import_external( - name = "com_google_errorprone_error_prone_annotations", - artifact = "com.google.errorprone:error_prone_annotations:2.3.4", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "baf7d6ea97ce606c53e11b6854ba5f2ce7ef5c24dddf0afa18d1260bd25b002c", - licenses = ["notice"], # Apache 2.0 - ) - -def com_google_guava_guava(): - jvm_maven_import_external( - name = "com_google_guava_guava", - artifact = "com.google.guava:guava:29.0-android", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "00ba22cb0e32610db7cf8ab4c20017c85d11788600734ff1d86995345eb5bc3b", - licenses = ["notice"], # Apache 2.0 - ) - -def com_google_guava_failureaccess(): - jvm_maven_import_external( - name = "com_google_guava_failureaccess", - artifact = "com.google.guava:failureaccess:1.0.1", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "a171ee4c734dd2da837e4b16be9df4661afab72a41adaf31eb84dfdaf936ca26", - licenses = ["notice"], # Apache 2.0 - ) - -def com_google_j2objc_j2objc_annotations(): - jvm_maven_import_external( - name = "com_google_j2objc_j2objc_annotations", - artifact = "com.google.j2objc:j2objc-annotations:1.3", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "21af30c92267bd6122c0e0b4d20cccb6641a37eaf956c6540ec471d584e64a7b", - licenses = ["notice"], # Apache 2.0 - ) - def com_google_protobuf(): # proto_library rules implicitly depend on @com_google_protobuf//:protoc, # which is the proto-compiler. @@ -280,33 +125,6 @@ def com_google_protobuf_javalite(): urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.12.0.zip"], ) -def com_google_truth_truth(): - jvm_maven_import_external( - name = "com_google_truth_truth", - artifact = "com.google.truth:truth:1.0.1", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "1ccf4334e7a94cf00a20a619b5462b53acf3274e00b70498bf5b28a3bc1be9b1", - licenses = ["notice"], # Apache 2.0 - ) - -def com_squareup_okhttp_okhttp(): - jvm_maven_import_external( - name = "com_squareup_okhttp_okhttp", - artifact = "com.squareup.okhttp:okhttp:2.7.4", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "c88be9af1509d5aeec9394a818c0fa08e26fad9d64ba134e6f977e0bb20cb114", - licenses = ["notice"], # Apache 2.0 - ) - -def com_squareup_okio_okio(): - jvm_maven_import_external( - name = "com_squareup_okio_okio", - artifact = "com.squareup.okio:okio:1.13.0", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "734269c3ebc5090e3b23566db558f421f0b4027277c79ad5d176b8ec168bb850", - licenses = ["notice"], # Apache 2.0 - ) - def io_grpc_grpc_proto(): http_archive( name = "io_grpc_grpc_proto", @@ -314,175 +132,3 @@ def io_grpc_grpc_proto(): strip_prefix = "grpc-proto-cf828d0e1155e5ea58b46d7184ee5596e03ddcb8", urls = ["https://github.com/grpc/grpc-proto/archive/cf828d0e1155e5ea58b46d7184ee5596e03ddcb8.zip"], ) - -def io_netty_netty_buffer(): - jvm_maven_import_external( - name = "io_netty_netty_buffer", - artifact = "io.netty:netty-buffer:4.1.51.Final", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "c3c3b710e1b5a8df3d60cd4602e0a743481d5e609e4aa852fa2629e4e412d245", - licenses = ["notice"], # Apache 2.0 - ) - -def io_netty_netty_codec(): - jvm_maven_import_external( - name = "io_netty_netty_codec", - artifact = "io.netty:netty-codec:4.1.51.Final", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "ff741aaa35f7048a6be7c700aa4851bf643917648ea5b7c0cbada2f3848c2bee", - licenses = ["notice"], # Apache 2.0 - ) - -def io_netty_netty_codec_http(): - jvm_maven_import_external( - name = "io_netty_netty_codec_http", - artifact = "io.netty:netty-codec-http:4.1.51.Final", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "e2fc9d846b77160d30df733bf9e88c6bcc589ab4a54719ac6c9195dd82865bea", - licenses = ["notice"], # Apache 2.0 - ) - -def io_netty_netty_codec_http2(): - jvm_maven_import_external( - name = "io_netty_netty_codec_http2", - artifact = "io.netty:netty-codec-http2:4.1.51.Final", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "48b0102de286e1f5528a17aed5b2a4fb35615b358fdde1b4d8702484f29cf87d", - licenses = ["notice"], # Apache 2.0 - ) - -def io_netty_netty_codec_socks(): - jvm_maven_import_external( - name = "io_netty_netty_codec_socks", - artifact = "io.netty:netty-codec-socks:4.1.51.Final", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "bd4d6f8917059a178eb4f94801f7dbfbde0f9f09accfdcf1addccd72081cf9a2", - licenses = ["notice"], # Apache 2.0 - ) - -def io_netty_netty_common(): - jvm_maven_import_external( - name = "io_netty_netty_common", - artifact = "io.netty:netty-common:4.1.51.Final", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "110e06515f43913a2bbac23e1aa78b7f59ae09d466b00af5fcf399a4f9af1b6b", - licenses = ["notice"], # Apache 2.0 - ) - -def io_netty_netty_handler(): - jvm_maven_import_external( - name = "io_netty_netty_handler", - artifact = "io.netty:netty-handler:4.1.51.Final", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "4461970f04f4d5eb9112ad94255ce1987394ce64de6c3c87690bf0865c936258", - licenses = ["notice"], # Apache 2.0 - ) - -def io_netty_netty_handler_proxy(): - jvm_maven_import_external( - name = "io_netty_netty_handler_proxy", - artifact = "io.netty:netty-handler-proxy:4.1.51.Final", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "8ed8f70c5c9591e9f168b7ae6b315944409ef404839204d972906e8e1de171dd", - licenses = ["notice"], # Apache 2.0 - ) - -def io_netty_netty_resolver(): - jvm_maven_import_external( - name = "io_netty_netty_resolver", - artifact = "io.netty:netty-resolver:4.1.51.Final", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "c8a77765e481fbf5906c596eb441de49096b354bcae0356b7404ac5e96399350", - licenses = ["notice"], # Apache 2.0 - ) - -def io_netty_netty_tcnative_boringssl_static(): - jvm_maven_import_external( - name = "io_netty_netty_tcnative_boringssl_static", - artifact = "io.netty:netty-tcnative-boringssl-static:2.0.31.Final", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "308e7e1f5faea3ff86bf689f6309b8605090cdd5186586a52e418bf48af93d68", - licenses = ["notice"], # Apache 2.0 - ) - -def io_netty_netty_transport(): - jvm_maven_import_external( - name = "io_netty_netty_transport", - artifact = "io.netty:netty-transport:4.1.51.Final", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "e5be259f35a246bf504ad93ea8f5df31872b5abebfb751380eab95d5dc840d44", - licenses = ["notice"], # Apache 2.0 - ) - -def io_netty_netty_transport_native_epoll_linux_x86_64(): - jvm_maven_import_external( - name = "io_netty_netty_transport_native_epoll_linux_x86_64", - artifact = "io.netty:netty-transport-native-epoll:jar:linux-x86_64:4.1.51.Final", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "8058a306f5b4d511f8d31e6700121359a659dd90fe3a0902d2b32e70bdc21a57", - licenses = ["notice"], # Apache 2.0 - ) - -def io_opencensus_opencensus_api(): - jvm_maven_import_external( - name = "io_opencensus_opencensus_api", - artifact = "io.opencensus:opencensus-api:0.24.0", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "f561b1cc2673844288e596ddf5bb6596868a8472fd2cb8993953fc5c034b2352", - licenses = ["notice"], # Apache 2.0 - ) - -def io_opencensus_opencensus_contrib_grpc_metrics(): - jvm_maven_import_external( - name = "io_opencensus_opencensus_contrib_grpc_metrics", - artifact = "io.opencensus:opencensus-contrib-grpc-metrics:0.24.0", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "875582e093f11950ad3f4a50b5fee33a008023f7d1e47820a1bef05d23b9ed42", - licenses = ["notice"], # Apache 2.0 - ) - -def io_perfmark_perfmark_api(): - jvm_maven_import_external( - name = "io_perfmark_perfmark_api", - artifact = "io.perfmark:perfmark-api:0.19.0", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "b734ba2149712409a44eabdb799f64768578fee0defe1418bb108fe32ea43e1a", - licenses = ["notice"], # Apache 2.0 - ) - -def junit_junit(): - jvm_maven_import_external( - name = "junit_junit", - artifact = "junit:junit:4.12", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "59721f0805e223d84b90677887d9ff567dc534d7c502ca903c0c2b17f05c116a", - licenses = ["notice"], # EPL 1.0 - ) - -def org_apache_commons_commons_lang3(): - jvm_maven_import_external( - name = "org_apache_commons_commons_lang3", - artifact = "org.apache.commons:commons-lang3:3.5", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "8ac96fc686512d777fca85e144f196cd7cfe0c0aec23127229497d1a38ff651c", - licenses = ["notice"], # Apache 2.0 - ) - -def org_apache_tomcat_annotations_api(): - # Use //stub:javax_annotation for neverlink=1 support. - jvm_maven_import_external( - name = "org_apache_tomcat_annotations_api", - artifact = "org.apache.tomcat:annotations-api:6.0.53", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "253829d3c12b7381d1044fc22c6436cff025fe0d459e4a329413e560a7d0dd13", - licenses = ["notice"], # Apache 2.0 - ) - -def org_codehaus_mojo_animal_sniffer_annotations(): - jvm_maven_import_external( - name = "org_codehaus_mojo_animal_sniffer_annotations", - artifact = "org.codehaus.mojo:animal-sniffer-annotations:1.18", - server_urls = ["https://repo.maven.apache.org/maven2/"], - artifact_sha256 = "47f05852b48ee9baefef80fa3d8cea60efa4753c0013121dd7fe5eef2e5c729d", - licenses = ["notice"], # MIT - ) From 09967c106067a6b3b883cebfbc722f2d078dbb6d Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Mon, 12 Oct 2020 18:18:59 -0700 Subject: [PATCH 84/86] xds: gate xDS timeout with env variable (v1.33.x backport) (#7504) (#7509) --- xds/src/main/java/io/grpc/xds/XdsNameResolver.java | 11 ++++++++--- .../test/java/io/grpc/xds/XdsNameResolverTest.java | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java index 534ac7f86fa..b6167e606ac 100644 --- a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java +++ b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java @@ -65,6 +65,9 @@ final class XdsNameResolver extends NameResolver { static final CallOptions.Key CLUSTER_SELECTION_KEY = CallOptions.Key.create("io.grpc.xds.CLUSTER_SELECTION_KEY"); + @VisibleForTesting + static boolean enableTimeout = + Boolean.parseBoolean(System.getenv("GRPC_XDS_EXPERIMENTAL_ENABLE_TIMEOUT")); private final XdsLogger logger; private final String authority; @@ -242,9 +245,11 @@ public Result selectConfig(PickSubchannelArgs args) { } } while (!retainCluster(cluster)); // TODO(chengyuanzhang): avoid service config generation and parsing for each call. - Map rawServiceConfig = - generateServiceConfigWithMethodTimeoutConfig( - selectedRoute.getRouteAction().getTimeoutNano()); + Map rawServiceConfig = Collections.emptyMap(); + if (enableTimeout) { + rawServiceConfig = generateServiceConfigWithMethodTimeoutConfig( + selectedRoute.getRouteAction().getTimeoutNano()); + } ConfigOrError parsedServiceConfig = serviceConfigParser.parseServiceConfig(rawServiceConfig); Object config = parsedServiceConfig.getConfig(); if (config == null) { diff --git a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java index 797bd45a561..b842ed85db4 100644 --- a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java @@ -116,6 +116,7 @@ XdsChannel createChannel(List servers) throws XdsInitializationExcep @Before public void setUp() { + XdsNameResolver.enableTimeout = true; Bootstrapper bootstrapper = new Bootstrapper() { @Override public BootstrapInfo readBootstrap() { From 7d8410f9e16d32b6ac5e0a923ba80a5858ae9247 Mon Sep 17 00:00:00 2001 From: "Penn (Dapeng) Zhang" Date: Tue, 20 Oct 2020 12:56:17 -0400 Subject: [PATCH 85/86] Update README etc to reference 1.33.0 --- README.md | 28 ++++++++++++------------ cronet/README.md | 2 +- documentation/android-channel-builder.md | 4 ++-- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 737fc40c1d6..791dee6e908 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,8 @@ For a guided tour, take a look at the [quick start guide](https://grpc.io/docs/languages/java/quickstart) or the more explanatory [gRPC basics](https://grpc.io/docs/languages/java/basics). -The [examples](https://github.com/grpc/grpc-java/tree/v1.32.1/examples) and the -[Android example](https://github.com/grpc/grpc-java/tree/v1.32.1/examples/android) +The [examples](https://github.com/grpc/grpc-java/tree/v1.33.0/examples) and the +[Android example](https://github.com/grpc/grpc-java/tree/v1.33.0/examples/android) are standalone projects that showcase the usage of gRPC. Download @@ -42,17 +42,17 @@ Download [the JARs][]. Or for Maven with non-Android, add to your `pom.xml`: io.grpc grpc-netty-shaded - 1.32.1 + 1.33.0 io.grpc grpc-protobuf - 1.32.1 + 1.33.0 io.grpc grpc-stub - 1.32.1 + 1.33.0 org.apache.tomcat @@ -64,23 +64,23 @@ Download [the JARs][]. Or for Maven with non-Android, add to your `pom.xml`: Or for Gradle with non-Android, add to your dependencies: ```gradle -implementation 'io.grpc:grpc-netty-shaded:1.32.1' -implementation 'io.grpc:grpc-protobuf:1.32.1' -implementation 'io.grpc:grpc-stub:1.32.1' +implementation 'io.grpc:grpc-netty-shaded:1.33.0' +implementation 'io.grpc:grpc-protobuf:1.33.0' +implementation 'io.grpc:grpc-stub:1.33.0' compileOnly 'org.apache.tomcat:annotations-api:6.0.53' // necessary for Java 9+ ``` For Android client, use `grpc-okhttp` instead of `grpc-netty-shaded` and `grpc-protobuf-lite` instead of `grpc-protobuf`: ```gradle -implementation 'io.grpc:grpc-okhttp:1.32.1' -implementation 'io.grpc:grpc-protobuf-lite:1.32.1' -implementation 'io.grpc:grpc-stub:1.32.1' +implementation 'io.grpc:grpc-okhttp:1.33.0' +implementation 'io.grpc:grpc-protobuf-lite:1.33.0' +implementation 'io.grpc:grpc-stub:1.33.0' compileOnly 'org.apache.tomcat:annotations-api:6.0.53' // necessary for Java 9+ ``` [the JARs]: -https://search.maven.org/search?q=g:io.grpc%20AND%20v:1.32.1 +https://search.maven.org/search?q=g:io.grpc%20AND%20v:1.33.0 Development snapshots are available in [Sonatypes's snapshot repository](https://oss.sonatype.org/content/repositories/snapshots/). @@ -112,7 +112,7 @@ For protobuf-based codegen integrated with the Maven build system, you can use com.google.protobuf:protoc:3.12.0:exe:${os.detected.classifier} grpc-java - io.grpc:protoc-gen-grpc-java:1.32.1:exe:${os.detected.classifier} + io.grpc:protoc-gen-grpc-java:1.33.0:exe:${os.detected.classifier} @@ -142,7 +142,7 @@ protobuf { } plugins { grpc { - artifact = 'io.grpc:protoc-gen-grpc-java:1.32.1' + artifact = 'io.grpc:protoc-gen-grpc-java:1.33.0' } } generateProtoTasks { diff --git a/cronet/README.md b/cronet/README.md index b752082c42f..530fbea71b0 100644 --- a/cronet/README.md +++ b/cronet/README.md @@ -26,7 +26,7 @@ In your app module's `build.gradle` file, include a dependency on both `grpc-cro Google Play Services Client Library for Cronet ``` -implementation 'io.grpc:grpc-cronet:1.32.1' +implementation 'io.grpc:grpc-cronet:1.33.0' implementation 'com.google.android.gms:play-services-cronet:16.0.0' ``` diff --git a/documentation/android-channel-builder.md b/documentation/android-channel-builder.md index 579f4f13210..b152c9286a0 100644 --- a/documentation/android-channel-builder.md +++ b/documentation/android-channel-builder.md @@ -36,8 +36,8 @@ In your `build.gradle` file, include a dependency on both `grpc-android` and `grpc-okhttp`: ``` -implementation 'io.grpc:grpc-android:1.32.1' -implementation 'io.grpc:grpc-okhttp:1.32.1' +implementation 'io.grpc:grpc-android:1.33.0' +implementation 'io.grpc:grpc-okhttp:1.33.0' ``` You also need permission to access the device's network state in your From bec6a8638de963597fd3dd264280c4309e339966 Mon Sep 17 00:00:00 2001 From: "Penn (Dapeng) Zhang" Date: Tue, 20 Oct 2020 13:00:48 -0400 Subject: [PATCH 86/86] Bump version to 1.33.0 --- build.gradle | 2 +- .../src/test/golden/TestDeprecatedService.java.txt | 2 +- compiler/src/test/golden/TestService.java.txt | 2 +- .../src/testLite/golden/TestDeprecatedService.java.txt | 2 +- compiler/src/testLite/golden/TestService.java.txt | 2 +- core/src/main/java/io/grpc/internal/GrpcUtil.java | 2 +- examples/android/clientcache/app/build.gradle | 10 +++++----- examples/android/helloworld/app/build.gradle | 8 ++++---- examples/android/routeguide/app/build.gradle | 8 ++++---- examples/android/strictmode/app/build.gradle | 8 ++++---- examples/build.gradle | 2 +- examples/example-alts/build.gradle | 2 +- examples/example-gauth/build.gradle | 2 +- examples/example-gauth/pom.xml | 4 ++-- examples/example-hostname/build.gradle | 2 +- examples/example-hostname/pom.xml | 4 ++-- examples/example-jwt-auth/build.gradle | 2 +- examples/example-jwt-auth/pom.xml | 4 ++-- examples/example-tls/build.gradle | 2 +- examples/example-tls/pom.xml | 4 ++-- examples/example-xds/build.gradle | 2 +- examples/pom.xml | 4 ++-- 22 files changed, 40 insertions(+), 40 deletions(-) diff --git a/build.gradle b/build.gradle index 67750629a2f..dff0b7f479a 100644 --- a/build.gradle +++ b/build.gradle @@ -18,7 +18,7 @@ subprojects { apply plugin: "net.ltgt.errorprone" group = "io.grpc" - version = "1.33.0-SNAPSHOT" // CURRENT_GRPC_VERSION + version = "1.33.0" // CURRENT_GRPC_VERSION repositories { maven { // The google mirror is less flaky than mavenCentral() diff --git a/compiler/src/test/golden/TestDeprecatedService.java.txt b/compiler/src/test/golden/TestDeprecatedService.java.txt index edceeae5cb6..1753171fb59 100644 --- a/compiler/src/test/golden/TestDeprecatedService.java.txt +++ b/compiler/src/test/golden/TestDeprecatedService.java.txt @@ -21,7 +21,7 @@ import static io.grpc.stub.ServerCalls.asyncUnimplementedUnaryCall; * */ @javax.annotation.Generated( - value = "by gRPC proto compiler (version 1.33.0-SNAPSHOT)", + value = "by gRPC proto compiler (version 1.33.0)", comments = "Source: grpc/testing/compiler/test.proto") @java.lang.Deprecated public final class TestDeprecatedServiceGrpc { diff --git a/compiler/src/test/golden/TestService.java.txt b/compiler/src/test/golden/TestService.java.txt index 033c93175dd..f6f7bf34571 100644 --- a/compiler/src/test/golden/TestService.java.txt +++ b/compiler/src/test/golden/TestService.java.txt @@ -21,7 +21,7 @@ import static io.grpc.stub.ServerCalls.asyncUnimplementedUnaryCall; * */ @javax.annotation.Generated( - value = "by gRPC proto compiler (version 1.33.0-SNAPSHOT)", + value = "by gRPC proto compiler (version 1.33.0)", comments = "Source: grpc/testing/compiler/test.proto") public final class TestServiceGrpc { diff --git a/compiler/src/testLite/golden/TestDeprecatedService.java.txt b/compiler/src/testLite/golden/TestDeprecatedService.java.txt index b7ce089ca2f..15804335d67 100644 --- a/compiler/src/testLite/golden/TestDeprecatedService.java.txt +++ b/compiler/src/testLite/golden/TestDeprecatedService.java.txt @@ -21,7 +21,7 @@ import static io.grpc.stub.ServerCalls.asyncUnimplementedUnaryCall; * */ @javax.annotation.Generated( - value = "by gRPC proto compiler (version 1.33.0-SNAPSHOT)", + value = "by gRPC proto compiler (version 1.33.0)", comments = "Source: grpc/testing/compiler/test.proto") @java.lang.Deprecated public final class TestDeprecatedServiceGrpc { diff --git a/compiler/src/testLite/golden/TestService.java.txt b/compiler/src/testLite/golden/TestService.java.txt index 4693d662b2c..fb24ab020a2 100644 --- a/compiler/src/testLite/golden/TestService.java.txt +++ b/compiler/src/testLite/golden/TestService.java.txt @@ -21,7 +21,7 @@ import static io.grpc.stub.ServerCalls.asyncUnimplementedUnaryCall; * */ @javax.annotation.Generated( - value = "by gRPC proto compiler (version 1.33.0-SNAPSHOT)", + value = "by gRPC proto compiler (version 1.33.0)", comments = "Source: grpc/testing/compiler/test.proto") public final class TestServiceGrpc { diff --git a/core/src/main/java/io/grpc/internal/GrpcUtil.java b/core/src/main/java/io/grpc/internal/GrpcUtil.java index 18a3ebb7271..ce46d8c9bf5 100644 --- a/core/src/main/java/io/grpc/internal/GrpcUtil.java +++ b/core/src/main/java/io/grpc/internal/GrpcUtil.java @@ -197,7 +197,7 @@ public byte[] parseAsciiString(byte[] serialized) { public static final Splitter ACCEPT_ENCODING_SPLITTER = Splitter.on(',').trimResults(); - private static final String IMPLEMENTATION_VERSION = "1.33.0-SNAPSHOT"; // CURRENT_GRPC_VERSION + private static final String IMPLEMENTATION_VERSION = "1.33.0"; // CURRENT_GRPC_VERSION /** * The default timeout in nanos for a keepalive ping request. diff --git a/examples/android/clientcache/app/build.gradle b/examples/android/clientcache/app/build.gradle index 084cc22aba0..6ae625419a6 100644 --- a/examples/android/clientcache/app/build.gradle +++ b/examples/android/clientcache/app/build.gradle @@ -30,7 +30,7 @@ android { protobuf { protoc { artifact = 'com.google.protobuf:protoc:3.12.0' } plugins { - grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION + grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.33.0' // CURRENT_GRPC_VERSION } } generateProtoTasks { @@ -50,12 +50,12 @@ dependencies { implementation 'com.android.support:appcompat-v7:27.0.2' // You need to build grpc-java to obtain these libraries below. - implementation 'io.grpc:grpc-okhttp:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-protobuf-lite:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-stub:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-okhttp:1.33.0' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-protobuf-lite:1.33.0' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-stub:1.33.0' // CURRENT_GRPC_VERSION implementation 'org.apache.tomcat:annotations-api:6.0.53' testImplementation 'junit:junit:4.12' testImplementation 'com.google.truth:truth:1.0.1' - testImplementation 'io.grpc:grpc-testing:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION + testImplementation 'io.grpc:grpc-testing:1.33.0' // CURRENT_GRPC_VERSION } diff --git a/examples/android/helloworld/app/build.gradle b/examples/android/helloworld/app/build.gradle index ab300b8428f..5211c63b6c1 100644 --- a/examples/android/helloworld/app/build.gradle +++ b/examples/android/helloworld/app/build.gradle @@ -28,7 +28,7 @@ android { protobuf { protoc { artifact = 'com.google.protobuf:protoc:3.12.0' } plugins { - grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION + grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.33.0' // CURRENT_GRPC_VERSION } } generateProtoTasks { @@ -48,8 +48,8 @@ dependencies { implementation 'com.android.support:appcompat-v7:27.0.2' // You need to build grpc-java to obtain these libraries below. - implementation 'io.grpc:grpc-okhttp:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-protobuf-lite:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-stub:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-okhttp:1.33.0' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-protobuf-lite:1.33.0' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-stub:1.33.0' // CURRENT_GRPC_VERSION implementation 'org.apache.tomcat:annotations-api:6.0.53' } diff --git a/examples/android/routeguide/app/build.gradle b/examples/android/routeguide/app/build.gradle index e1a9d728800..c0621d62bc4 100644 --- a/examples/android/routeguide/app/build.gradle +++ b/examples/android/routeguide/app/build.gradle @@ -28,7 +28,7 @@ android { protobuf { protoc { artifact = 'com.google.protobuf:protoc:3.12.0' } plugins { - grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION + grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.33.0' // CURRENT_GRPC_VERSION } } generateProtoTasks { @@ -48,8 +48,8 @@ dependencies { implementation 'com.android.support:appcompat-v7:27.0.2' // You need to build grpc-java to obtain these libraries below. - implementation 'io.grpc:grpc-okhttp:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-protobuf-lite:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-stub:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-okhttp:1.33.0' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-protobuf-lite:1.33.0' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-stub:1.33.0' // CURRENT_GRPC_VERSION implementation 'org.apache.tomcat:annotations-api:6.0.53' } diff --git a/examples/android/strictmode/app/build.gradle b/examples/android/strictmode/app/build.gradle index 35cca7019a9..7dee456fb1f 100644 --- a/examples/android/strictmode/app/build.gradle +++ b/examples/android/strictmode/app/build.gradle @@ -29,7 +29,7 @@ android { protobuf { protoc { artifact = 'com.google.protobuf:protoc:3.12.0' } plugins { - grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION + grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.33.0' // CURRENT_GRPC_VERSION } } generateProtoTasks { @@ -49,8 +49,8 @@ dependencies { implementation 'com.android.support:appcompat-v7:28.0.0' // You need to build grpc-java to obtain these libraries below. - implementation 'io.grpc:grpc-okhttp:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-protobuf-lite:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-stub:1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-okhttp:1.33.0' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-protobuf-lite:1.33.0' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-stub:1.33.0' // CURRENT_GRPC_VERSION implementation 'org.apache.tomcat:annotations-api:6.0.53' } diff --git a/examples/build.gradle b/examples/build.gradle index aacb920e557..4b6c733151e 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -22,7 +22,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.33.0' // CURRENT_GRPC_VERSION def protobufVersion = '3.12.0' def protocVersion = protobufVersion diff --git a/examples/example-alts/build.gradle b/examples/example-alts/build.gradle index 9e71cdb9f50..e01598187f3 100644 --- a/examples/example-alts/build.gradle +++ b/examples/example-alts/build.gradle @@ -23,7 +23,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.33.0' // CURRENT_GRPC_VERSION def protocVersion = '3.12.0' dependencies { diff --git a/examples/example-gauth/build.gradle b/examples/example-gauth/build.gradle index 5283b4a19bc..6542df94351 100644 --- a/examples/example-gauth/build.gradle +++ b/examples/example-gauth/build.gradle @@ -23,7 +23,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.33.0' // CURRENT_GRPC_VERSION def protobufVersion = '3.12.0' def protocVersion = protobufVersion diff --git a/examples/example-gauth/pom.xml b/examples/example-gauth/pom.xml index 9c81ea6d1af..a255df8a0dd 100644 --- a/examples/example-gauth/pom.xml +++ b/examples/example-gauth/pom.xml @@ -6,13 +6,13 @@ jar - 1.33.0-SNAPSHOT + 1.33.0 example-gauth https://github.com/grpc/grpc-java UTF-8 - 1.33.0-SNAPSHOT + 1.33.0 3.12.0 1.7 diff --git a/examples/example-hostname/build.gradle b/examples/example-hostname/build.gradle index 2515e189d64..f7f44ab430e 100644 --- a/examples/example-hostname/build.gradle +++ b/examples/example-hostname/build.gradle @@ -21,7 +21,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.33.0' // CURRENT_GRPC_VERSION def protobufVersion = '3.12.0' dependencies { diff --git a/examples/example-hostname/pom.xml b/examples/example-hostname/pom.xml index 9b616a0a3fb..a01948f90ce 100644 --- a/examples/example-hostname/pom.xml +++ b/examples/example-hostname/pom.xml @@ -6,13 +6,13 @@ jar - 1.33.0-SNAPSHOT + 1.33.0 example-hostname https://github.com/grpc/grpc-java UTF-8 - 1.33.0-SNAPSHOT + 1.33.0 3.12.0 1.7 diff --git a/examples/example-jwt-auth/build.gradle b/examples/example-jwt-auth/build.gradle index a258c98c484..d827dcdf017 100644 --- a/examples/example-jwt-auth/build.gradle +++ b/examples/example-jwt-auth/build.gradle @@ -22,7 +22,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.33.0' // CURRENT_GRPC_VERSION def protobufVersion = '3.12.0' def protocVersion = protobufVersion diff --git a/examples/example-jwt-auth/pom.xml b/examples/example-jwt-auth/pom.xml index 8aa08257741..c6064be0a00 100644 --- a/examples/example-jwt-auth/pom.xml +++ b/examples/example-jwt-auth/pom.xml @@ -7,13 +7,13 @@ jar - 1.33.0-SNAPSHOT + 1.33.0 example-jwt-auth https://github.com/grpc/grpc-java UTF-8 - 1.33.0-SNAPSHOT + 1.33.0 3.12.0 3.12.0 diff --git a/examples/example-tls/build.gradle b/examples/example-tls/build.gradle index bebb2946ce2..1fb5859bdca 100644 --- a/examples/example-tls/build.gradle +++ b/examples/example-tls/build.gradle @@ -23,7 +23,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.33.0' // CURRENT_GRPC_VERSION def nettyTcNativeVersion = '2.0.31.Final' def protocVersion = '3.12.0' diff --git a/examples/example-tls/pom.xml b/examples/example-tls/pom.xml index fbbad150547..08f948edf94 100644 --- a/examples/example-tls/pom.xml +++ b/examples/example-tls/pom.xml @@ -6,13 +6,13 @@ jar - 1.33.0-SNAPSHOT + 1.33.0 example-tls https://github.com/grpc/grpc-java UTF-8 - 1.33.0-SNAPSHOT + 1.33.0 3.12.0 2.0.31.Final diff --git a/examples/example-xds/build.gradle b/examples/example-xds/build.gradle index fab5c9452b2..ccb3a68803c 100644 --- a/examples/example-xds/build.gradle +++ b/examples/example-xds/build.gradle @@ -18,7 +18,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.33.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.33.0' // CURRENT_GRPC_VERSION dependencies { // This example's client is the same as the helloworld client. We depend on the helloworld diff --git a/examples/pom.xml b/examples/pom.xml index 843d7da2adb..e7d66e501c6 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -6,13 +6,13 @@ jar - 1.33.0-SNAPSHOT + 1.33.0 examples https://github.com/grpc/grpc-java UTF-8 - 1.33.0-SNAPSHOT + 1.33.0 3.12.0 3.12.0