Change the HttpStreamFactory::JobController to mark alternative service broken until default network changes.
Change the HttpStreamFactory::JobController to mark alternative service broken until default network changes iff TCP succeeds, QUIC fails on the defaut network but succeeds on the alternate network. The HttpServerProperties will clear the brokenness until default network changes.
Bug: 790547
Change-Id: Ib33def42b474898879bf9e39567632ed273c5ca2
Reviewed-on: https://chromium-review.googlesource.com/1212142
Commit-Queue: Zhongyi Shi <[email protected]>
Reviewed-by: Ryan Hamilton <[email protected]>
Cr-Commit-Position: refs/heads/master@{#593486}diff --git a/net/quic/quic_network_transaction_unittest.cc b/net/quic/quic_network_transaction_unittest.cc
index f5ee80c..26f05ae 100644
--- a/net/quic/quic_network_transaction_unittest.cc
+++ b/net/quic/quic_network_transaction_unittest.cc
@@ -816,6 +816,18 @@
socket_factory_.AddSSLSocketDataProvider(&ssl_data_);
}
+ void SetUpTestForRetryConnectionOnAlternateNetwork() {
+ session_params_.quic_migrate_sessions_on_network_change_v2 = true;
+ session_params_.quic_migrate_sessions_early_v2 = true;
+ session_params_.quic_retry_on_alternate_network_before_handshake = true;
+ scoped_mock_change_notifier_.reset(new ScopedMockNetworkChangeNotifier());
+ MockNetworkChangeNotifier* mock_ncn =
+ scoped_mock_change_notifier_->mock_network_change_notifier();
+ mock_ncn->ForceNetworkHandlesSupported();
+ mock_ncn->SetConnectedNetworksList(
+ {kDefaultNetworkForTests, kNewNetworkForTests});
+ }
+
// Fetches two non-cryptographic URL requests via a HTTPS proxy with a QUIC
// alternative proxy. Verifies that if the alternative proxy job returns
// |error_code|, the request is fetched successfully by the main job.
@@ -919,6 +931,7 @@
BoundTestNetLog net_log_;
std::vector<std::unique_ptr<StaticSocketDataProvider>> hanging_data_;
SSLSocketDataProvider ssl_data_;
+ std::unique_ptr<ScopedMockNetworkChangeNotifier> scoped_mock_change_notifier_;
private:
void SendRequestAndExpectQuicResponseMaybeFromProxy(
@@ -2136,6 +2149,394 @@
EXPECT_TRUE(details.quic_port_migration_detected);
}
+// This test verifies that a new QUIC connection will be attempted on the
+// alternate network if the original QUIC connection fails with idle timeout
+// before handshake is confirmed. If TCP succeeds and QUIC fails on the
+// alternate network as well, QUIC is marked as broken and the brokenness will
+// not expire when default network changes.
+TEST_P(QuicNetworkTransactionTest, QuicFailsOnBothNetworksWhileTCPSucceeds) {
+ SetUpTestForRetryConnectionOnAlternateNetwork();
+
+ std::string request_data;
+ client_maker_.SetEncryptionLevel(quic::ENCRYPTION_INITIAL);
+ client_maker_.SetLongHeaderType(quic::ZERO_RTT_PROTECTED);
+
+ // The request will initially go out over QUIC.
+ MockQuicData quic_data;
+ quic_data.AddRead(SYNCHRONOUS, ERR_IO_PENDING); // Hanging read
+ int packet_num = 1;
+ quic_data.AddWrite(SYNCHRONOUS,
+ client_maker_.MakeDummyCHLOPacket(packet_num++)); // CHLO
+ // Retranmit the handshake messages.
+ quic_data.AddWrite(SYNCHRONOUS,
+ client_maker_.MakeDummyCHLOPacket(packet_num++));
+ quic_data.AddWrite(SYNCHRONOUS,
+ client_maker_.MakeDummyCHLOPacket(packet_num++));
+ quic_data.AddWrite(SYNCHRONOUS,
+ client_maker_.MakeDummyCHLOPacket(packet_num++));
+ quic_data.AddWrite(SYNCHRONOUS,
+ client_maker_.MakeDummyCHLOPacket(packet_num++));
+ // TODO(zhongyi): remove condition check once b/115926584 is fixed.
+ if (version_ <= quic::QUIC_VERSION_39) {
+ quic_data.AddWrite(SYNCHRONOUS,
+ client_maker_.MakeDummyCHLOPacket(packet_num++));
+ }
+ // After timeout, connection will be closed with QUIC_NETWORK_IDLE_TIMEOUT.
+ quic_data.AddWrite(SYNCHRONOUS,
+ client_maker_.MakeConnectionClosePacket(
+ packet_num++, true, quic::QUIC_NETWORK_IDLE_TIMEOUT,
+ "No recent network activity."));
+ quic_data.AddSocketDataToFactory(&socket_factory_);
+
+ // Add successful TCP data so that TCP job will succeed.
+ MockWrite http_writes[] = {
+ MockWrite(SYNCHRONOUS, 0, "GET / HTTP/1.1\r\n"),
+ MockWrite(SYNCHRONOUS, 1, "Host: mail.example.org\r\n"),
+ MockWrite(SYNCHRONOUS, 2, "Connection: keep-alive\r\n\r\n")};
+
+ MockRead http_reads[] = {
+ MockRead(SYNCHRONOUS, 3, "HTTP/1.1 200 OK\r\n"),
+ MockRead(SYNCHRONOUS, 4, kQuicAlternativeServiceHeader),
+ MockRead(SYNCHRONOUS, 5, "TCP succeeds"), MockRead(SYNCHRONOUS, OK, 6)};
+ SequencedSocketData http_data(http_reads, http_writes);
+ socket_factory_.AddSocketDataProvider(&http_data);
+ socket_factory_.AddSSLSocketDataProvider(&ssl_data_);
+
+ // Add data for the second QUIC connection to fail.
+ MockQuicData quic_data2;
+ quic_data2.AddRead(SYNCHRONOUS, ERR_IO_PENDING);
+ quic_data2.AddWrite(SYNCHRONOUS, ERR_ADDRESS_UNREACHABLE); // Write error.
+ quic_data2.AddSocketDataToFactory(&socket_factory_);
+
+ // Resolve the host resolution synchronously.
+ host_resolver_.set_synchronous_mode(true);
+ host_resolver_.rules()->AddIPLiteralRule("mail.example.org", "192.168.0.1",
+ "");
+ HostResolver::RequestInfo info(HostPortPair("mail.example.org", 443));
+ AddressList address;
+ std::unique_ptr<HostResolver::Request> request;
+ int rv = host_resolver_.Resolve(info, DEFAULT_PRIORITY, &address,
+ CompletionOnceCallback(), &request,
+ net_log_.bound());
+ EXPECT_THAT(rv, IsOk());
+
+ CreateSession();
+ session_->quic_stream_factory()->set_require_confirmation(true);
+ // Use a TestTaskRunner to avoid waiting in real time for timeouts.
+ scoped_refptr<TestTaskRunner> quic_task_runner_(new TestTaskRunner(&clock_));
+ QuicStreamFactoryPeer::SetAlarmFactory(
+ session_->quic_stream_factory(),
+ std::make_unique<QuicChromiumAlarmFactory>(quic_task_runner_.get(),
+ &clock_));
+ // Add alternate protocol mapping to race QUIC and TCP.
+ // QUIC connection requires handshake to be confirmed and sends CHLO to the
+ // peer.
+ AddQuicAlternateProtocolMapping(
+ MockCryptoClientStream::COLD_START_WITH_CHLO_SENT);
+
+ HttpNetworkTransaction trans(DEFAULT_PRIORITY, session_.get());
+ TestCompletionCallback callback;
+ rv = trans.Start(&request_, callback.callback(), net_log_.bound());
+ EXPECT_THAT(rv, IsError(ERR_IO_PENDING));
+
+ // Pump the message loop to get the request started.
+ // Request will be served with TCP job.
+ base::RunLoop().RunUntilIdle();
+ EXPECT_THAT(callback.WaitForResult(), IsOk());
+ CheckResponseData(&trans, "TCP succeeds");
+
+ // Fire the retransmission alarm, from this point, connection will idle
+ // timeout after 4 seconds.
+ // TODO(zhongyi): remove manually firing the alarm when b/116260560 is fixed.
+ quic_task_runner_->RunNextTask();
+
+ // Fast forward to idle timeout the original connection. A new connection will
+ // be kicked off on the alternate network.
+ quic_task_runner_->FastForwardBy(quic::QuicTime::Delta::FromSeconds(4));
+ ASSERT_TRUE(quic_data.AllReadDataConsumed());
+ ASSERT_TRUE(quic_data.AllWriteDataConsumed());
+
+ // Run the message loop to execute posted tasks, which will report job status.
+ base::RunLoop().RunUntilIdle();
+
+ // Verify that QUIC is marked as broken.
+ ExpectBrokenAlternateProtocolMapping();
+
+ // Deliver a message to notify the new network becomes default, the brokenness
+ // will not expire as QUIC is broken on both networks.
+ scoped_mock_change_notifier_->mock_network_change_notifier()
+ ->NotifyNetworkMadeDefault(kNewNetworkForTests);
+ ExpectBrokenAlternateProtocolMapping();
+
+ ASSERT_TRUE(quic_data2.AllReadDataConsumed());
+ ASSERT_TRUE(quic_data2.AllWriteDataConsumed());
+}
+
+// This test verifies that a new QUIC connection will be attempted on the
+// alternate network if the original QUIC connection fails with idle timeout
+// before handshake is confirmed. If TCP succeeds and QUIC succeeds on the
+// alternate network, QUIC is marked as broken. The brokenness will expire when
+// the default network changes.
+TEST_P(QuicNetworkTransactionTest, RetryOnAlternateNetworkWhileTCPSucceeds) {
+ SetUpTestForRetryConnectionOnAlternateNetwork();
+
+ std::string request_data;
+ client_maker_.SetEncryptionLevel(quic::ENCRYPTION_INITIAL);
+ client_maker_.SetLongHeaderType(quic::ZERO_RTT_PROTECTED);
+
+ // The request will initially go out over QUIC.
+ MockQuicData quic_data;
+ quic_data.AddRead(SYNCHRONOUS, ERR_IO_PENDING); // Hanging read
+ int packet_num = 1;
+ quic_data.AddWrite(SYNCHRONOUS,
+ client_maker_.MakeDummyCHLOPacket(packet_num++)); // CHLO
+ // Retranmit the handshake messages.
+ quic_data.AddWrite(SYNCHRONOUS,
+ client_maker_.MakeDummyCHLOPacket(packet_num++));
+ quic_data.AddWrite(SYNCHRONOUS,
+ client_maker_.MakeDummyCHLOPacket(packet_num++));
+ quic_data.AddWrite(SYNCHRONOUS,
+ client_maker_.MakeDummyCHLOPacket(packet_num++));
+ quic_data.AddWrite(SYNCHRONOUS,
+ client_maker_.MakeDummyCHLOPacket(packet_num++));
+ // TODO(zhongyi): remove condition check once b/115926584 is fixed.
+ if (version_ <= quic::QUIC_VERSION_39) {
+ quic_data.AddWrite(SYNCHRONOUS,
+ client_maker_.MakeDummyCHLOPacket(packet_num++));
+ }
+ // After timeout, connection will be closed with QUIC_NETWORK_IDLE_TIMEOUT.
+ quic_data.AddWrite(SYNCHRONOUS,
+ client_maker_.MakeConnectionClosePacket(
+ packet_num++, true, quic::QUIC_NETWORK_IDLE_TIMEOUT,
+ "No recent network activity."));
+ quic_data.AddSocketDataToFactory(&socket_factory_);
+
+ // Add successful TCP data so that TCP job will succeed.
+ MockWrite http_writes[] = {
+ MockWrite(SYNCHRONOUS, 0, "GET / HTTP/1.1\r\n"),
+ MockWrite(SYNCHRONOUS, 1, "Host: mail.example.org\r\n"),
+ MockWrite(SYNCHRONOUS, 2, "Connection: keep-alive\r\n\r\n")};
+
+ MockRead http_reads[] = {
+ MockRead(SYNCHRONOUS, 3, "HTTP/1.1 200 OK\r\n"),
+ MockRead(SYNCHRONOUS, 4, kQuicAlternativeServiceHeader),
+ MockRead(SYNCHRONOUS, 5, "TCP succeeds"), MockRead(SYNCHRONOUS, OK, 6)};
+ SequencedSocketData http_data(http_reads, http_writes);
+ socket_factory_.AddSocketDataProvider(&http_data);
+ socket_factory_.AddSSLSocketDataProvider(&ssl_data_);
+
+ // Quic connection will be retried on the alternate network after the initial
+ // one fails on the default network.
+ MockQuicData quic_data2;
+ quic::QuicStreamOffset header_stream_offset = 0;
+ quic_data2.AddRead(SYNCHRONOUS, ERR_IO_PENDING); // Handing read.
+ quic_data2.AddWrite(SYNCHRONOUS,
+ client_maker_.MakeDummyCHLOPacket(1)); // CHLO
+
+ client_maker_.SetEncryptionLevel(quic::ENCRYPTION_FORWARD_SECURE);
+ quic_data2.AddWrite(SYNCHRONOUS,
+ ConstructInitialSettingsPacket(2, &header_stream_offset));
+ quic_data2.AddSocketDataToFactory(&socket_factory_);
+
+ // Resolve the host resolution synchronously.
+ host_resolver_.set_synchronous_mode(true);
+ host_resolver_.rules()->AddIPLiteralRule("mail.example.org", "192.168.0.1",
+ "");
+ HostResolver::RequestInfo info(HostPortPair("mail.example.org", 443));
+ AddressList address;
+ std::unique_ptr<HostResolver::Request> request;
+ int rv = host_resolver_.Resolve(info, DEFAULT_PRIORITY, &address,
+ CompletionOnceCallback(), &request,
+ net_log_.bound());
+ EXPECT_THAT(rv, IsOk());
+
+ CreateSession();
+ session_->quic_stream_factory()->set_require_confirmation(true);
+ // Use a TestTaskRunner to avoid waiting in real time for timeouts.
+ scoped_refptr<TestTaskRunner> quic_task_runner_(new TestTaskRunner(&clock_));
+ QuicStreamFactoryPeer::SetAlarmFactory(
+ session_->quic_stream_factory(),
+ std::make_unique<QuicChromiumAlarmFactory>(quic_task_runner_.get(),
+ &clock_));
+ // Add alternate protocol mapping to race QUIC and TCP.
+ // QUIC connection requires handshake to be confirmed and sends CHLO to the
+ // peer.
+ AddQuicAlternateProtocolMapping(
+ MockCryptoClientStream::COLD_START_WITH_CHLO_SENT);
+
+ HttpNetworkTransaction trans(DEFAULT_PRIORITY, session_.get());
+ TestCompletionCallback callback;
+ rv = trans.Start(&request_, callback.callback(), net_log_.bound());
+ EXPECT_THAT(rv, IsError(ERR_IO_PENDING));
+
+ // Pump the message loop to get the request started.
+ // Request will be served with TCP job.
+ base::RunLoop().RunUntilIdle();
+ EXPECT_THAT(callback.WaitForResult(), IsOk());
+ CheckResponseData(&trans, "TCP succeeds");
+
+ // Fire the retransmission alarm, after which connection will idle
+ // timeout after 4 seconds.
+ // TODO(zhongyi): remove manually firing the alarm when b/116260560 is fixed.
+ quic_task_runner_->RunNextTask();
+
+ // Fast forward to idle timeout the original connection. A new connection will
+ // be kicked off on the alternate network.
+ quic_task_runner_->FastForwardBy(quic::QuicTime::Delta::FromSeconds(4));
+ ASSERT_TRUE(quic_data.AllReadDataConsumed());
+ ASSERT_TRUE(quic_data.AllWriteDataConsumed());
+
+ // The second connection hasn't finish handshake, verify that QUIC is not
+ // marked as broken.
+ ExpectQuicAlternateProtocolMapping();
+ // Explicitly confirm the handshake on the second connection.
+ crypto_client_stream_factory_.last_stream()->SendOnCryptoHandshakeEvent(
+ quic::QuicSession::HANDSHAKE_CONFIRMED);
+ // Run message loop to execute posted tasks, which will notify JoController
+ // about the orphaned job status.
+ base::RunLoop().RunUntilIdle();
+
+ // Verify that QUIC is marked as broken.
+ ExpectBrokenAlternateProtocolMapping();
+
+ // Deliver a message to notify the new network becomes default, the previous
+ // brokenness will be clear as the brokenness is bond with old default
+ // network.
+ scoped_mock_change_notifier_->mock_network_change_notifier()
+ ->NotifyNetworkMadeDefault(kNewNetworkForTests);
+ ExpectQuicAlternateProtocolMapping();
+
+ ASSERT_TRUE(quic_data2.AllReadDataConsumed());
+ ASSERT_TRUE(quic_data2.AllWriteDataConsumed());
+}
+
+// This test verifies that a new QUIC connection will be attempted on the
+// alternate network if the original QUIC connection fails with idle timeout
+// before handshake is confirmed. If TCP doesn't succeed but QUIC on the
+// alternative network succeeds, QUIC is not marked as broken.
+TEST_P(QuicNetworkTransactionTest, RetryOnAlternateNetworkWhileTCPHanging) {
+ SetUpTestForRetryConnectionOnAlternateNetwork();
+
+ std::string request_data;
+ client_maker_.SetEncryptionLevel(quic::ENCRYPTION_INITIAL);
+ client_maker_.SetLongHeaderType(quic::ZERO_RTT_PROTECTED);
+
+ // The request will initially go out over QUIC.
+ MockQuicData quic_data;
+ quic_data.AddRead(SYNCHRONOUS, ERR_IO_PENDING); // Hanging read
+ int packet_num = 1;
+ quic_data.AddWrite(SYNCHRONOUS,
+ client_maker_.MakeDummyCHLOPacket(packet_num++)); // CHLO
+ // Retranmit the handshake messages.
+ quic_data.AddWrite(SYNCHRONOUS,
+ client_maker_.MakeDummyCHLOPacket(packet_num++));
+ quic_data.AddWrite(SYNCHRONOUS,
+ client_maker_.MakeDummyCHLOPacket(packet_num++));
+ quic_data.AddWrite(SYNCHRONOUS,
+ client_maker_.MakeDummyCHLOPacket(packet_num++));
+ quic_data.AddWrite(SYNCHRONOUS,
+ client_maker_.MakeDummyCHLOPacket(packet_num++));
+ // TODO(zhongyi): remove condition check once b/115926584 is fixed, i.e.,
+ // quic_fix_has_pending_crypto_data is introduced and enabled.
+ if (version_ <= quic::QUIC_VERSION_39) {
+ quic_data.AddWrite(SYNCHRONOUS,
+ client_maker_.MakeDummyCHLOPacket(packet_num++));
+ }
+ // After timeout, connection will be closed with QUIC_NETWORK_IDLE_TIMEOUT.
+ quic_data.AddWrite(SYNCHRONOUS,
+ client_maker_.MakeConnectionClosePacket(
+ packet_num++, true, quic::QUIC_NETWORK_IDLE_TIMEOUT,
+ "No recent network activity."));
+ quic_data.AddSocketDataToFactory(&socket_factory_);
+
+ // Add hanging TCP data so that TCP job will never succeeded.
+ AddHangingNonAlternateProtocolSocketData();
+
+ // Quic connection will then be retried on the alternate network.
+ MockQuicData quic_data2;
+ quic::QuicStreamOffset header_stream_offset = 0;
+ quic_data2.AddWrite(SYNCHRONOUS,
+ client_maker_.MakeDummyCHLOPacket(1)); // CHLO
+
+ client_maker_.SetEncryptionLevel(quic::ENCRYPTION_FORWARD_SECURE);
+ quic_data2.AddWrite(SYNCHRONOUS,
+ ConstructInitialSettingsPacket(2, &header_stream_offset));
+ quic_data2.AddWrite(
+ SYNCHRONOUS,
+ ConstructClientRequestHeadersPacket(
+ 3, GetNthClientInitiatedStreamId(0), true, true,
+ GetRequestHeaders("GET", "https", "/"), &header_stream_offset));
+ quic_data2.AddRead(ASYNC, ConstructServerResponseHeadersPacket(
+ 1, GetNthClientInitiatedStreamId(0), false,
+ false, GetResponseHeaders("200 OK")));
+ quic_data2.AddRead(
+ ASYNC, ConstructServerDataPacket(2, GetNthClientInitiatedStreamId(0),
+ false, true, 0, "hello!"));
+ quic_data2.AddWrite(SYNCHRONOUS, ConstructClientAckPacket(4, 2, 1, 1));
+ quic_data2.AddRead(SYNCHRONOUS, ERR_IO_PENDING); // No more data to read
+ quic_data2.AddSocketDataToFactory(&socket_factory_);
+
+ // Resolve the host resolution synchronously.
+ host_resolver_.set_synchronous_mode(true);
+ host_resolver_.rules()->AddIPLiteralRule("mail.example.org", "192.168.0.1",
+ "");
+ HostResolver::RequestInfo info(HostPortPair("mail.example.org", 443));
+ AddressList address;
+ std::unique_ptr<HostResolver::Request> request;
+ int rv = host_resolver_.Resolve(info, DEFAULT_PRIORITY, &address,
+ CompletionOnceCallback(), &request,
+ net_log_.bound());
+ EXPECT_THAT(rv, IsOk());
+
+ CreateSession();
+ session_->quic_stream_factory()->set_require_confirmation(true);
+ // Use a TestTaskRunner to avoid waiting in real time for timeouts.
+ scoped_refptr<TestTaskRunner> quic_task_runner_(new TestTaskRunner(&clock_));
+ QuicStreamFactoryPeer::SetAlarmFactory(
+ session_->quic_stream_factory(),
+ std::make_unique<QuicChromiumAlarmFactory>(quic_task_runner_.get(),
+ &clock_));
+ // Add alternate protocol mapping to race QUIC and TCP.
+ // QUIC connection requires handshake to be confirmed and sends CHLO to the
+ // peer.
+ AddQuicAlternateProtocolMapping(
+ MockCryptoClientStream::COLD_START_WITH_CHLO_SENT);
+
+ HttpNetworkTransaction trans(DEFAULT_PRIORITY, session_.get());
+ TestCompletionCallback callback;
+ rv = trans.Start(&request_, callback.callback(), net_log_.bound());
+ EXPECT_THAT(rv, IsError(ERR_IO_PENDING));
+
+ // Pump the message loop to get the request started.
+ base::RunLoop().RunUntilIdle();
+ quic_task_runner_->RunNextTask();
+
+ // Fast forward to idle timeout the original connection. A new connection will
+ // be kicked off on the alternate network.
+ quic_task_runner_->FastForwardBy(quic::QuicTime::Delta::FromSeconds(4));
+ ASSERT_TRUE(quic_data.AllReadDataConsumed());
+ ASSERT_TRUE(quic_data.AllWriteDataConsumed());
+
+ // Verify that QUIC is not marked as broken.
+ ExpectQuicAlternateProtocolMapping();
+ // Explicitly confirm the handshake on the second connection.
+ crypto_client_stream_factory_.last_stream()->SendOnCryptoHandshakeEvent(
+ quic::QuicSession::HANDSHAKE_CONFIRMED);
+
+ // Read the response.
+ EXPECT_THAT(callback.WaitForResult(), IsOk());
+ CheckResponseData(&trans, "hello!");
+ // Verify that QUIC is not marked as broken.
+ ExpectQuicAlternateProtocolMapping();
+
+ // Deliver a message to notify the new network becomes default.
+ scoped_mock_change_notifier_->mock_network_change_notifier()
+ ->NotifyNetworkMadeDefault(kNewNetworkForTests);
+ ExpectQuicAlternateProtocolMapping();
+ ASSERT_TRUE(quic_data2.AllReadDataConsumed());
+ ASSERT_TRUE(quic_data2.AllWriteDataConsumed());
+}
+
// Verify that if a QUIC connection times out, the QuicHttpStream will
// return QUIC_PROTOCOL_ERROR.
TEST_P(QuicNetworkTransactionTest, TimeoutAfterHandshakeConfirmed) {
