blob: caf2400cb0b741e02df44d420d2bacc123883dbd [file] [log] [blame]
license.botbf09a502008-08-24 00:55:551// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
initial.commit09911bf2008-07-26 23:55:294
5#include "chrome/browser/safe_browsing/protocol_manager.h"
6
[email protected]484fce42008-10-01 00:37:187#include "base/file_version_info.h"
initial.commit09911bf2008-07-26 23:55:298#include "base/histogram.h"
9#include "base/logging.h"
10#include "base/message_loop.h"
[email protected]05f9b682008-09-29 22:18:0111#include "base/rand_util.h"
initial.commit09911bf2008-07-26 23:55:2912#include "base/string_util.h"
[email protected]05f9b682008-09-29 22:18:0113#include "base/sys_info.h"
initial.commit09911bf2008-07-26 23:55:2914#include "base/task.h"
15#include "base/timer.h"
16#include "chrome/browser/profile.h"
17#include "chrome/browser/safe_browsing/protocol_parser.h"
initial.commit09911bf2008-07-26 23:55:2918#include "chrome/browser/safe_browsing/safe_browsing_service.h"
initial.commit09911bf2008-07-26 23:55:2919#include "chrome/common/env_vars.h"
initial.commit09911bf2008-07-26 23:55:2920#include "chrome/common/stl_util-inl.h"
21#include "net/base/base64.h"
22#include "net/base/load_flags.h"
23
24
25// Maximum time, in seconds, from start up before we must issue an update query.
[email protected]05f9b682008-09-29 22:18:0126static const int kSbTimerStartIntervalSec = 5 * 60;
initial.commit09911bf2008-07-26 23:55:2927
28// Update URL for querying about the latest set of chunk updates.
29static const char* const kSbUpdateUrl =
[email protected]484fce42008-10-01 00:37:1830 "http://safebrowsing.clients.google.com/safebrowsing/downloads?client=%s&appver=%s&pver=2.1";
initial.commit09911bf2008-07-26 23:55:2931
32// GetHash request URL for retrieving full hashes.
33static const char* const kSbGetHashUrl =
[email protected]484fce42008-10-01 00:37:1834 "http://safebrowsing.clients.google.com/safebrowsing/gethash?client=%s&appver=%s&pver=2.1";
initial.commit09911bf2008-07-26 23:55:2935
36// New MAC client key requests URL.
37static const char* const kSbNewKeyUrl =
[email protected]484fce42008-10-01 00:37:1838 "https://sb-ssl.google.com/safebrowsing/newkey?client=%s&appver=%s&pver=2.1";
initial.commit09911bf2008-07-26 23:55:2939
[email protected]f1da1262008-08-31 23:03:5840#if defined(GOOGLE_CHROME_BUILD)
41static const char* const kSbClientName = "googlechrome";
42#else
43static const char* const kSbClientName = "chromium";
44#endif
initial.commit09911bf2008-07-26 23:55:2945
46// Maximum back off multiplier.
47static const int kSbMaxBackOff = 8;
48
49
initial.commit09911bf2008-07-26 23:55:2950// SafeBrowsingProtocolManager implementation ----------------------------------
51
52SafeBrowsingProtocolManager::SafeBrowsingProtocolManager(
53 SafeBrowsingService* sb_service,
54 MessageLoop* notify_loop,
55 const std::string& client_key,
56 const std::string& wrapped_key)
57 : sb_service_(sb_service),
58 request_type_(NO_REQUEST),
59 update_error_count_(0),
60 gethash_error_count_(0),
61 update_back_off_mult_(1),
62 gethash_back_off_mult_(1),
63 next_update_sec_(-1),
64 update_state_(FIRST_REQUEST),
65 initial_request_(true),
66 chunk_pending_to_write_(false),
67 notify_loop_(notify_loop),
68 client_key_(client_key),
69 wrapped_key_(wrapped_key) {
70 // Set the backoff multiplier fuzz to a random value between 0 and 1.
[email protected]05f9b682008-09-29 22:18:0171 back_off_fuzz_ = static_cast<float>(base::RandDouble());
initial.commit09911bf2008-07-26 23:55:2972
73 // The first update must happen between 0-5 minutes of start up.
[email protected]05f9b682008-09-29 22:18:0174 next_update_sec_ = base::RandInt(60, kSbTimerStartIntervalSec);
[email protected]484fce42008-10-01 00:37:1875
76 scoped_ptr<FileVersionInfo> version_info(
77 FileVersionInfo::CreateFileVersionInfoForCurrentModule());
78 if (!version_info.get())
79 version_ = "0.1";
80 else
81 version_ = WideToASCII(version_info->product_version());
initial.commit09911bf2008-07-26 23:55:2982}
83
84SafeBrowsingProtocolManager::~SafeBrowsingProtocolManager() {
initial.commit09911bf2008-07-26 23:55:2985 // Delete in-progress SafeBrowsing requests.
86 STLDeleteContainerPairFirstPointers(hash_requests_.begin(),
87 hash_requests_.end());
88 hash_requests_.clear();
89}
90
91// Public API used by the SafeBrowsingService ----------------------------------
92
93// We can only have one update or chunk request outstanding, but there may be
94// multiple GetHash requests pending since we don't want to serialize them and
95// slow down the user.
96void SafeBrowsingProtocolManager::GetFullHash(
97 SafeBrowsingService::SafeBrowsingCheck* check,
98 const std::vector<SBPrefix>& prefixes) {
99 // If we are in GetHash backoff, we need to check if we're past the next
100 // allowed time. If we are, we can proceed with the request. If not, we are
101 // required to return empty results (i.e. treat the page as safe).
102 if (gethash_error_count_ && Time::Now() <= next_gethash_time_) {
103 std::vector<SBFullHashResult> full_hashes;
[email protected]200abc32008-09-05 01:44:33104 sb_service_->HandleGetHashResults(check, full_hashes, false);
initial.commit09911bf2008-07-26 23:55:29105 return;
106 }
107
108 std::string url = StringPrintf(kSbGetHashUrl,
109 kSbClientName,
[email protected]484fce42008-10-01 00:37:18110 version_.c_str());
initial.commit09911bf2008-07-26 23:55:29111 if (!client_key_.empty()) {
112 url.append("&wrkey=");
113 url.append(wrapped_key_);
114 }
115
116 GURL gethash_url(url);
117 URLFetcher* fetcher = new URLFetcher(gethash_url, URLFetcher::POST, this);
118 hash_requests_[fetcher] = check;
119
120 std::string get_hash;
121 SafeBrowsingProtocolParser parser;
122 parser.FormatGetHash(prefixes, &get_hash);
123
124 fetcher->set_load_flags(net::LOAD_DISABLE_CACHE);
125 fetcher->set_request_context(Profile::GetDefaultRequestContext());
[email protected]d36e3c8e2008-08-29 23:42:20126 fetcher->set_upload_data("text/plain", get_hash);
initial.commit09911bf2008-07-26 23:55:29127 fetcher->Start();
128}
129
130void SafeBrowsingProtocolManager::GetNextUpdate() {
131 if (initial_request_) {
132 if (client_key_.empty() || wrapped_key_.empty()) {
133 IssueKeyRequest();
134 return;
135 } else {
136 initial_request_ = false;
137 }
138 }
139
140 if (!request_.get())
141 IssueUpdateRequest();
142}
143
144// URLFetcher::Delegate implementation -----------------------------------------
145
146// All SafeBrowsing request responses are handled here.
147// TODO(paulg): Clarify with the SafeBrowsing team whether a failed parse of a
148// chunk should retry the download and parse of that chunk (and
149// what back off / how many times to try), and if that effects the
150// update back off. For now, a failed parse of the chunk means we
151// drop it. This isn't so bad because the next UPDATE_REQUEST we
152// do will report all the chunks we have. If that chunk is still
153// required, the SafeBrowsing servers will tell us to get it again.
154void SafeBrowsingProtocolManager::OnURLFetchComplete(
155 const URLFetcher* source,
156 const GURL& url,
157 const URLRequestStatus& status,
158 int response_code,
159 const ResponseCookies& cookies,
160 const std::string& data) {
161 scoped_ptr<const URLFetcher> fetcher;
162 bool parsed_ok = true;
163 bool must_back_off = false; // Reduce SafeBrowsing service query frequency.
164
165 HashRequests::iterator it = hash_requests_.find(source);
166 if (it != hash_requests_.end()) {
167 // GetHash response.
168 fetcher.reset(it->first);
169 SafeBrowsingService::SafeBrowsingCheck* check = it->second;
170 std::vector<SBFullHashResult> full_hashes;
[email protected]200abc32008-09-05 01:44:33171 bool can_cache = false;
initial.commit09911bf2008-07-26 23:55:29172 if (response_code == 200 || response_code == 204) {
[email protected]200abc32008-09-05 01:44:33173 can_cache = true;
initial.commit09911bf2008-07-26 23:55:29174 gethash_error_count_ = 0;
175 gethash_back_off_mult_ = 1;
176 bool re_key = false;
177 SafeBrowsingProtocolParser parser;
178 parsed_ok = parser.ParseGetHash(data.data(),
179 static_cast<int>(data.length()),
180 client_key_,
181 &re_key,
182 &full_hashes);
183 if (!parsed_ok) {
184 // If we fail to parse it, we must still inform the SafeBrowsingService
185 // so that it doesn't hold up the user's request indefinitely. Not sure
186 // what to do at that point though!
187 full_hashes.clear();
188 } else {
189 if (re_key)
190 HandleReKey();
191 }
192 } else if (response_code >= 300) {
193 HandleGetHashError();
194 SB_DLOG(INFO) << "SafeBrowsing GetHash request for: " << source->url()
195 << ", failed with error: " << response_code;
196 }
197
198 // Call back the SafeBrowsingService with full_hashes, even if there was a
199 // parse error or an error response code (in which case full_hashes will be
200 // empty). We can't block the user regardless of the error status.
[email protected]200abc32008-09-05 01:44:33201 sb_service_->HandleGetHashResults(check, full_hashes, can_cache);
initial.commit09911bf2008-07-26 23:55:29202
203 hash_requests_.erase(it);
204 } else {
205 // Update, chunk or key response.
206 DCHECK(source == request_.get());
207 fetcher.reset(request_.release());
208
209 if (response_code == 200) {
210 // We have data from the SafeBrowsing service.
211 parsed_ok = HandleServiceResponse(source->url(),
212 data.data(),
213 static_cast<int>(data.length()));
214 if (!parsed_ok) {
215 SB_DLOG(INFO) << "SafeBrowsing request for: " << source->url()
216 << "failed parse.";
[email protected]22717d1e2008-10-15 21:55:32217 must_back_off = true;
218 chunk_request_urls_.clear();
initial.commit09911bf2008-07-26 23:55:29219 }
220
[email protected]22717d1e2008-10-15 21:55:32221 if (request_type_ == CHUNK_REQUEST && parsed_ok) {
222 chunk_request_urls_.pop_front();
initial.commit09911bf2008-07-26 23:55:29223 } else if (request_type_ == GETKEY_REQUEST && initial_request_) {
224 // This is the first request we've made this session. Now that we have
225 // the keys, do the regular update request.
226 initial_request_ = false;
227 GetNextUpdate();
228 return;
229 }
230 } else if (response_code >= 300) {
231 // The SafeBrowsing service error: back off.
232 must_back_off = true;
233 if (request_type_ == CHUNK_REQUEST)
234 chunk_request_urls_.clear();
235 SB_DLOG(INFO) << "SafeBrowsing request for: " << source->url()
236 << ", failed with error: " << response_code;
237 }
238 }
239
240 // Schedule a new update request if we've finished retrieving all the chunks
241 // from the previous update. We treat the update request and the chunk URLs it
242 // contains as an atomic unit as far as back off is concerned.
243 if (chunk_request_urls_.empty() &&
244 (request_type_ == CHUNK_REQUEST || request_type_ == UPDATE_REQUEST))
245 ScheduleNextUpdate(must_back_off);
246
247 // Get the next chunk if available.
248 IssueChunkRequest();
249}
250
251bool SafeBrowsingProtocolManager::HandleServiceResponse(const GURL& url,
252 const char* data,
253 int length) {
254 SafeBrowsingProtocolParser parser;
255
256 switch (request_type_) {
257 case UPDATE_REQUEST: {
258 int next_update_sec = -1;
259 bool re_key = false;
260 bool reset = false;
261 std::vector<SBChunkDelete>* chunk_deletes =
262 new std::vector<SBChunkDelete>;
263 std::vector<ChunkUrl> chunk_urls;
264 if (!parser.ParseUpdate(data, length, client_key_,
265 &next_update_sec, &re_key,
266 &reset, chunk_deletes, &chunk_urls)) {
267 delete chunk_deletes;
268 return false;
269 }
270
271 last_update_ = Time::Now();
272
273 if (update_state_ == FIRST_REQUEST)
274 update_state_ = SECOND_REQUEST;
275 else if (update_state_ == SECOND_REQUEST)
276 update_state_ = NORMAL_REQUEST;
277
278 // New time for the next update.
279 if (next_update_sec > 0) {
280 next_update_sec_ = next_update_sec;
281 } else if (update_state_ == SECOND_REQUEST) {
[email protected]05f9b682008-09-29 22:18:01282 next_update_sec_ = base::RandInt(15 * 60, 45 * 60);
initial.commit09911bf2008-07-26 23:55:29283 }
284
285 // We need to request a new set of keys for MAC.
286 if (re_key)
287 HandleReKey();
288
289 // New chunks to download.
290 if (!chunk_urls.empty()) {
291 for (size_t i = 0; i < chunk_urls.size(); ++i)
292 chunk_request_urls_.push_back(chunk_urls[i]);
293 }
294
295 // Handle the case were the SafeBrowsing service tells us to dump our
296 // database.
297 if (reset) {
298 sb_service_->ResetDatabase();
299 return true;
300 }
301
302 // Chunks to delete from our storage.
303 if (!chunk_deletes->empty())
304 sb_service_->HandleChunkDelete(chunk_deletes);
305
306 break;
307 }
308 case CHUNK_REQUEST: {
309 // Find list name from url.
310 std::string url_path = url.ExtractFileName();
311 if (url_path.empty())
312 return false;
313
314 std::string::size_type pos = url_path.find_first_of('_');
315 if (pos == std::string::npos)
316 return false;
317
318 const ChunkUrl chunk_url = chunk_request_urls_.front();
319 DCHECK(url.spec().find(chunk_url.url) != std::string::npos);
320
321 bool re_key = false;
322 std::deque<SBChunk>* chunks = new std::deque<SBChunk>;
323 if (!parser.ParseChunk(data, length,
324 client_key_, chunk_url.mac,
325 &re_key, chunks)) {
326#ifndef NDEBUG
327 std::string data_str;
328 data_str.assign(data, length);
329 std::string encoded_chunk;
[email protected]a9bb6f692008-07-30 16:40:10330 net::Base64Encode(data, &encoded_chunk);
initial.commit09911bf2008-07-26 23:55:29331 SB_DLOG(INFO) << "ParseChunk error for chunk: " << chunk_url.url
332 << ", client_key: " << client_key_
333 << ", wrapped_key: " << wrapped_key_
334 << ", mac: " << chunk_url.mac
335 << ", Base64Encode(data): " << encoded_chunk
336 << ", length: " << length;
337#endif
338 safe_browsing_util::FreeChunks(chunks);
339 delete chunks;
340 return false;
341 }
342
343 if (re_key)
344 HandleReKey();
345
346 if (chunks->empty()) {
347 delete chunks;
348 } else {
349 chunk_pending_to_write_ = true;
350 std::string list_name(url_path, 0, pos);
351 sb_service_->HandleChunk(list_name, chunks);
352 }
353
354 break;
355 }
356 case GETKEY_REQUEST: {
357 std::string client_key, wrapped_key;
358 if (!parser.ParseNewKey(data, length, &client_key, &wrapped_key))
359 return false;
360
361 client_key_ = client_key;
362 wrapped_key_ = wrapped_key;
363 notify_loop_->PostTask(FROM_HERE, NewRunnableMethod(
364 sb_service_, &SafeBrowsingService::OnNewMacKeys, client_key_,
365 wrapped_key_));
366 break;
367 }
368
369 default:
370 return false;
371 }
372
373 return true;
374}
375
376void SafeBrowsingProtocolManager::Initialize() {
377 // Don't want to hit the safe browsing servers on build/chrome bots.
[email protected]05f9b682008-09-29 22:18:01378 if (base::SysInfo::HasEnvVar(env_vars::kHeadless))
initial.commit09911bf2008-07-26 23:55:29379 return;
380
381 ScheduleNextUpdate(false /* no back off */);
382}
383
384void SafeBrowsingProtocolManager::ScheduleNextUpdate(bool back_off) {
385 DCHECK(next_update_sec_ > 0);
386
[email protected]2d316662008-09-03 18:18:14387 // Unschedule any current timer.
388 update_timer_.Stop();
initial.commit09911bf2008-07-26 23:55:29389
390 // Reschedule with the new update.
391 const int next_update = GetNextUpdateTime(back_off);
[email protected]2d316662008-09-03 18:18:14392 update_timer_.Start(TimeDelta::FromMilliseconds(next_update), this,
393 &SafeBrowsingProtocolManager::GetNextUpdate);
initial.commit09911bf2008-07-26 23:55:29394}
395
396// According to section 5 of the SafeBrowsing protocol specification, we must
397// back off after a certain number of errors. We only change 'next_update_sec_'
398// when we receive a response from the SafeBrowsing service.
399int SafeBrowsingProtocolManager::GetNextUpdateTime(bool back_off) {
400 int next = next_update_sec_;
401 if (back_off) {
402 next = GetNextBackOffTime(&update_error_count_, &update_back_off_mult_);
403 } else {
404 // Successful response means error reset.
405 update_error_count_ = 0;
406 update_back_off_mult_ = 1;
407 }
408 return next * 1000; // milliseconds
409}
410
411int SafeBrowsingProtocolManager::GetNextBackOffTime(int* error_count,
412 int* multiplier) {
413 DCHECK(multiplier && error_count);
414 (*error_count)++;
415 if (*error_count > 1 && *error_count < 6) {
416 int next = static_cast<int>(*multiplier * (1 + back_off_fuzz_) * 30 * 60);
417 *multiplier *= 2;
418 if (*multiplier > kSbMaxBackOff)
419 *multiplier = kSbMaxBackOff;
420 return next;
421 }
422
423 if (*error_count >= 6)
424 return 60 * 60 * 8; // 8 hours
425
426 return 60; // 1 minute
427}
428
429// This request requires getting a list of all the chunks for each list from the
430// database asynchronously. The request will be issued when we're called back in
431// OnGetChunksComplete.
432// TODO(paulg): We should get this at start up and maintain a ChunkRange cache
433// to avoid hitting the database with each update request. On the
434// otherhand, this request will only occur ~20-30 minutes so there
435// isn't that much overhead. Measure!
436void SafeBrowsingProtocolManager::IssueUpdateRequest() {
437 request_type_ = UPDATE_REQUEST;
438 sb_service_->GetAllChunks();
439}
440
441void SafeBrowsingProtocolManager::IssueChunkRequest() {
442 // We are only allowed to have one request outstanding at any time. Also,
443 // don't get the next url until the previous one has been written to disk so
444 // that we don't use too much memory.
445 if (request_.get() || chunk_request_urls_.empty() || chunk_pending_to_write_)
446 return;
447
448 ChunkUrl next_chunk = chunk_request_urls_.front();
449 DCHECK(!next_chunk.url.empty());
450 if (!StartsWithASCII(next_chunk.url, "http://", false) &&
451 !StartsWithASCII(next_chunk.url, "https://", false))
452 next_chunk.url = "http://" + next_chunk.url;
453 GURL chunk_url(next_chunk.url);
454 request_type_ = CHUNK_REQUEST;
455 request_.reset(new URLFetcher(chunk_url, URLFetcher::GET, this));
456 request_->set_load_flags(net::LOAD_DISABLE_CACHE);
457 request_->set_request_context(Profile::GetDefaultRequestContext());
458 request_->Start();
459}
460
461void SafeBrowsingProtocolManager::IssueKeyRequest() {
462 GURL key_url(StringPrintf(kSbNewKeyUrl,
463 kSbClientName,
[email protected]484fce42008-10-01 00:37:18464 version_.c_str()));
initial.commit09911bf2008-07-26 23:55:29465 request_type_ = GETKEY_REQUEST;
466 request_.reset(new URLFetcher(key_url, URLFetcher::GET, this));
467 request_->set_load_flags(net::LOAD_DISABLE_CACHE);
468 request_->set_request_context(Profile::GetDefaultRequestContext());
469 request_->Start();
470}
471
472void SafeBrowsingProtocolManager::OnGetChunksComplete(
473 const std::vector<SBListChunkRanges>& lists, bool database_error) {
474 DCHECK(request_type_ == UPDATE_REQUEST);
475
476 if (database_error) {
477 ScheduleNextUpdate(false);
478 return;
479 }
480
481 const bool use_mac = !client_key_.empty();
482
483 // Format our stored chunks:
484 std::string list_data;
485 bool found_malware = false;
486 bool found_phishing = false;
487 for (size_t i = 0; i < lists.size(); ++i) {
488 list_data.append(FormatList(lists[i], use_mac));
489 if (lists[i].name == "goog-phish-shavar")
490 found_phishing = true;
491
492 if (lists[i].name == "goog-malware-shavar")
493 found_malware = true;
494 }
495
496 // If we have an empty database, let the server know we want data for these
497 // lists.
498 if (!found_phishing)
499 list_data.append(FormatList(SBListChunkRanges("goog-phish-shavar"),
500 use_mac));
501
502 if (!found_malware)
503 list_data.append(FormatList(SBListChunkRanges("goog-malware-shavar"),
504 use_mac));
505
506 std::string url = StringPrintf(kSbUpdateUrl,
507 kSbClientName,
[email protected]484fce42008-10-01 00:37:18508 version_.c_str());
initial.commit09911bf2008-07-26 23:55:29509 if (use_mac) {
510 url.append("&wrkey=");
511 url.append(wrapped_key_);
512 }
513
514 GURL update_url(url);
515 request_.reset(new URLFetcher(update_url, URLFetcher::POST, this));
516 request_->set_load_flags(net::LOAD_DISABLE_CACHE);
517 request_->set_request_context(Profile::GetDefaultRequestContext());
518 request_->set_upload_data("text/plain", list_data);
519 request_->Start();
520}
521
522void SafeBrowsingProtocolManager::OnChunkInserted() {
523 chunk_pending_to_write_ = false;
524
525 if (chunk_request_urls_.empty()) {
526 UMA_HISTOGRAM_LONG_TIMES(L"SB.Update", Time::Now() - last_update_);
[email protected]aad08752008-10-02 22:13:41527 sb_service_->UpdateFinished();
initial.commit09911bf2008-07-26 23:55:29528 } else {
529 IssueChunkRequest();
530 }
531}
532
533// static
534std::string SafeBrowsingProtocolManager::FormatList(
535 const SBListChunkRanges& list, bool use_mac) {
536 std::string formatted_results;
537 formatted_results.append(list.name);
538 formatted_results.append(";");
539 if (!list.adds.empty()) {
540 formatted_results.append("a:" + list.adds);
541 if (!list.subs.empty() || use_mac)
542 formatted_results.append(":");
543 }
544 if (!list.subs.empty()) {
545 formatted_results.append("s:" + list.subs);
546 if (use_mac)
547 formatted_results.append(":");
548 }
549 if (use_mac)
550 formatted_results.append("mac");
551 formatted_results.append("\n");
552
553 return formatted_results;
554}
555
556void SafeBrowsingProtocolManager::HandleReKey() {
557 client_key_.clear();
558 wrapped_key_.clear();
559 IssueKeyRequest();
560}
561
562void SafeBrowsingProtocolManager::HandleGetHashError() {
563 int next = GetNextBackOffTime(&gethash_error_count_, &gethash_back_off_mult_);
564 next_gethash_time_ = Time::Now() + TimeDelta::FromSeconds(next);
565}