|
15 | 15 | */ |
16 | 16 | package com.google.cloud.bigtable.gaxx.reframing; |
17 | 17 |
|
| 18 | +import static com.google.common.truth.Truth.assertWithMessage; |
| 19 | + |
18 | 20 | import com.google.api.gax.rpc.StreamController; |
19 | 21 | import com.google.cloud.bigtable.gaxx.testing.FakeStreamingApi.ServerStreamingStashCallable; |
20 | 22 | import com.google.cloud.bigtable.gaxx.testing.FakeStreamingApi.ServerStreamingStashCallable.StreamControllerStash; |
| 23 | +import com.google.cloud.bigtable.gaxx.testing.MockStreamingApi; |
21 | 24 | import com.google.cloud.bigtable.gaxx.testing.MockStreamingApi.MockResponseObserver; |
22 | 25 | import com.google.cloud.bigtable.gaxx.testing.MockStreamingApi.MockServerStreamingCall; |
23 | 26 | import com.google.cloud.bigtable.gaxx.testing.MockStreamingApi.MockServerStreamingCallable; |
|
27 | 30 | import com.google.common.collect.ImmutableList; |
28 | 31 | import com.google.common.collect.Queues; |
29 | 32 | import com.google.common.truth.Truth; |
| 33 | +import java.util.ArrayList; |
30 | 34 | import java.util.Arrays; |
| 35 | +import java.util.List; |
31 | 36 | import java.util.Queue; |
| 37 | +import java.util.concurrent.Callable; |
32 | 38 | import java.util.concurrent.CancellationException; |
| 39 | +import java.util.concurrent.CompletableFuture; |
33 | 40 | import java.util.concurrent.CountDownLatch; |
34 | 41 | import java.util.concurrent.ExecutionException; |
35 | 42 | import java.util.concurrent.ExecutorService; |
@@ -431,6 +438,120 @@ public String pop() { |
431 | 438 | Truth.assertThat(finalError.getSuppressed()[0].getCause()).isSameInstanceAs(fakeCancelError); |
432 | 439 | } |
433 | 440 |
|
| 441 | + /** |
| 442 | + * Test race between a request() and onComplete (b/295866356). This will stress the concurrency |
| 443 | + * primitives in deliver() by running a many iterations across many threads. Some race conditions |
| 444 | + * are very subtle and are very rare, so bugs in the implementation would present themselves as |
| 445 | + * flakes in this test. All flakes of this test should be investigated as a failure. |
| 446 | + */ |
| 447 | + @Test |
| 448 | + public void testRequestAndCompleteRaceCondition() throws Throwable { |
| 449 | + int concurrency = 20; |
| 450 | + int iterations = 20_000; |
| 451 | + |
| 452 | + ExecutorService executor = Executors.newFixedThreadPool(concurrency); |
| 453 | + |
| 454 | + List<Future<?>> results = new ArrayList<>(); |
| 455 | + |
| 456 | + for (int i = 0; i < concurrency; i++) { |
| 457 | + Future<?> result = |
| 458 | + executor.submit( |
| 459 | + (Callable<Void>) |
| 460 | + () -> { |
| 461 | + for (int j = 0; j < iterations; j++) { |
| 462 | + requestAndCompleteRaceConditionIteration(); |
| 463 | + } |
| 464 | + return null; |
| 465 | + }); |
| 466 | + results.add(result); |
| 467 | + } |
| 468 | + |
| 469 | + executor.shutdown(); |
| 470 | + |
| 471 | + for (Future<?> result : results) { |
| 472 | + try { |
| 473 | + result.get(); |
| 474 | + } catch (ExecutionException e) { |
| 475 | + throw e.getCause(); |
| 476 | + } |
| 477 | + } |
| 478 | + } |
| 479 | + |
| 480 | + private static void requestAndCompleteRaceConditionIteration() |
| 481 | + throws InterruptedException, ExecutionException { |
| 482 | + MockStreamingApi.MockResponseObserver<String> observer = |
| 483 | + new MockStreamingApi.MockResponseObserver<>(false); |
| 484 | + ReframingResponseObserver<String, String> underTest = |
| 485 | + new ReframingResponseObserver<>( |
| 486 | + observer, new ReframingResponseObserverTest.DasherizingReframer(1)); |
| 487 | + |
| 488 | + // This is intentionally not a Phaser, the Phaser seems to drastically reduce the reproduction |
| 489 | + // rate of the |
| 490 | + // original race condition. |
| 491 | + CountDownLatch readySignal = new CountDownLatch(2); |
| 492 | + CompletableFuture<Void> startSignal = new CompletableFuture<>(); |
| 493 | + |
| 494 | + ExecutorService executor = Executors.newFixedThreadPool(2); |
| 495 | + |
| 496 | + Future<Void> f1 = |
| 497 | + executor.submit( |
| 498 | + () -> { |
| 499 | + // no setup, tell controller thread we are ready and wait for the start signal |
| 500 | + readySignal.countDown(); |
| 501 | + startSignal.get(); |
| 502 | + |
| 503 | + // Race start |
| 504 | + underTest.onComplete(); |
| 505 | + // Race end |
| 506 | + |
| 507 | + return null; |
| 508 | + }); |
| 509 | + |
| 510 | + Future<Void> f2 = |
| 511 | + executor.submit( |
| 512 | + () -> { |
| 513 | + // Setup before race - simulate that the ServerStream iterator got one row and is now |
| 514 | + // checking if there |
| 515 | + // is another. This is the lead up to the race with grpc's onComplete |
| 516 | + underTest.onStart( |
| 517 | + new StreamController() { |
| 518 | + @Override |
| 519 | + public void cancel() {} |
| 520 | + |
| 521 | + @Override |
| 522 | + public void disableAutoInboundFlowControl() {} |
| 523 | + |
| 524 | + @Override |
| 525 | + public void request(int count) {} |
| 526 | + }); |
| 527 | + observer.getController().request(1); |
| 528 | + underTest.onResponse("moo"); |
| 529 | + |
| 530 | + // Setup complete, tell controller thread we are ready and wait for the start signal |
| 531 | + readySignal.countDown(); |
| 532 | + startSignal.get(); |
| 533 | + |
| 534 | + // Race start |
| 535 | + observer.getController().request(1); |
| 536 | + // Race end |
| 537 | + |
| 538 | + return null; |
| 539 | + }); |
| 540 | + executor.shutdown(); |
| 541 | + |
| 542 | + // Wait for worker setup |
| 543 | + readySignal.await(); |
| 544 | + // Tell workers to race |
| 545 | + startSignal.complete(null); |
| 546 | + |
| 547 | + // Wait workers to finish |
| 548 | + f1.get(); |
| 549 | + f2.get(); |
| 550 | + |
| 551 | + // the outer observer should be told of the completion of rpc |
| 552 | + assertWithMessage("outer observer should not hang").that(observer.isDone()).isTrue(); |
| 553 | + } |
| 554 | + |
434 | 555 | /** |
435 | 556 | * A simple implementation of a {@link Reframer}. The input string is split by dash, and the |
436 | 557 | * output is concatenated by dashes. The test can verify M:N behavior by adjusting the |
|
0 commit comments