Finish conversion from bloom filter to prefix set in safe browsing.

chrome/browser/safe_browsing/safe_browsing_database.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/browser/safe_browsing/safe_browsing_database.h"
 
 #include <algorithm>
 #include <iterator>
 
 #include "base/bind.h"
 #include "base/file_util.h"
 #include "base/message_loop.h"
 #include "base/metrics/histogram.h"
 #include "base/metrics/stats_counters.h"
 #include "base/process_util.h"
 #include "base/time.h"
-#include "chrome/browser/safe_browsing/bloom_filter.h"
 #include "chrome/browser/safe_browsing/prefix_set.h"
 #include "chrome/browser/safe_browsing/safe_browsing_store_file.h"
 #include "content/public/browser/browser_thread.h"
 #include "crypto/sha2.h"
 #include "googleurl/src/gurl.h"
 
 #if defined(OS_MACOSX)
 #include "base/mac/mac_util.h"
 #endif
 
@@ skipping 229 matching lines @@
   if (subs_deleted > 0)
     UMA_HISTOGRAM_COUNTS("SB2.DownloadBinhashSubsDeleted", subs_deleted);
 }
 
 // Order |SBAddFullHash| on the prefix part.  |SBAddPrefixLess()| from
 // safe_browsing_store.h orders on both chunk-id and prefix.
 bool SBAddFullHashPrefixLess(const SBAddFullHash& a, const SBAddFullHash& b) {
   return a.full_hash.prefix < b.full_hash.prefix;
 }
 
-// Track what LoadBloomFilterOrPrefixSet() loaded.
-enum FilterLoad {
-  FILTER_LOAD,                 // All calls.
-  FILTER_LOADED_PREFIX_SET,    // Cases loaded from prefix set.
-  FILTER_LOADED_BLOOM_FILTER,  // Cases loaded from bloom filter.
-
-  // Memory space for histograms is determined by the max.  ALWAYS ADD
-  // NEW VALUES BEFORE THIS ONE.
-  FILTER_LOAD_MAX
-};
-
-void RecordFilterLoad(FilterLoad event_type) {
-  UMA_HISTOGRAM_ENUMERATION("SB2.FilterLoad", event_type,
-                            FILTER_LOAD_MAX);
-}
-
 // This code always checks for non-zero file size.  This helper makes
 // that less verbose.
 int64 GetFileSizeOrZero(const FilePath& file_path) {
   int64 size_64;
   if (!file_util::GetFileSize(file_path, &size_64))
     return 0;
   return size_64;
 }
 
 }  // namespace
@@ skipping 131 matching lines @@
 
 void SafeBrowsingDatabaseNew::Init(const FilePath& filename_base) {
   DCHECK_EQ(creation_loop_, MessageLoop::current());
   // Ensure we haven't been run before.
   DCHECK(browse_filename_.empty());
   DCHECK(download_filename_.empty());
   DCHECK(csd_whitelist_filename_.empty());
   DCHECK(download_whitelist_filename_.empty());
 
   browse_filename_ = BrowseDBFilename(filename_base);
-  bloom_filter_filename_ = BloomFilterForFilename(browse_filename_);
   prefix_set_filename_ = PrefixSetForFilename(browse_filename_);
 
   browse_store_->Init(
       browse_filename_,
       base::Bind(&SafeBrowsingDatabaseNew::HandleCorruptDatabase,
                  base::Unretained(this)));
   DVLOG(1) << "Init browse store: " << browse_filename_.value();
 
   {
     // NOTE: There is no need to grab the lock in this function, since
     // until it returns, there are no pointers to this class on other
     // threads.  Then again, that means there is no possibility of
     // contention on the lock...
     base::AutoLock locked(lookup_lock_);
     full_browse_hashes_.clear();
     pending_browse_hashes_.clear();
-    LoadBloomFilterOrPrefixSet();
+    LoadPrefixSet();
   }
 
   if (download_store_.get()) {
     download_filename_ = DownloadDBFilename(filename_base);
     download_store_->Init(
         download_filename_,
         base::Bind(&SafeBrowsingDatabaseNew::HandleCorruptDatabase,
                    base::Unretained(this)));
     DVLOG(1) << "Init download store: " << download_filename_.value();
   }
@@ skipping 42 matching lines @@
   // reset.  Perhaps inline |Delete()|?
   if (!Delete())
     return false;
 
   // Reset objects in memory.
   {
     base::AutoLock locked(lookup_lock_);
     full_browse_hashes_.clear();
     pending_browse_hashes_.clear();
     prefix_miss_cache_.clear();
-    browse_bloom_filter_ = NULL;
     prefix_set_.reset();
   }
   // Wants to acquire the lock itself.
   WhitelistEverything(&csd_whitelist_);
   WhitelistEverything(&download_whitelist_);
 
   return true;
 }
 
 // TODO(lzheng): Remove matching_list, it is not used anywhere.
@@ skipping 10 matching lines @@
 
   std::vector<SBFullHash> full_hashes;
   BrowseFullHashesToCheck(url, false, &full_hashes);
   if (full_hashes.empty())
     return false;
 
   // This function is called on the I/O thread, prevent changes to
   // filter and caches.
   base::AutoLock locked(lookup_lock_);
 
-  // TODO(shess): During transition, users will have a bloom filter
-  // but no prefix set until first update, after which they'll have a
-  // prefix set but no bloom filter.
-  const bool use_prefix_set = prefix_set_.get() != NULL;
-  if (!use_prefix_set && !browse_bloom_filter_.get())
+  // |prefix_set_| is empty until it is either read from disk, or the
+  // first update populates it.  Bail out without a hit if not yet
+  // available.
+  if (!prefix_set_.get())
     return false;
 
   size_t miss_count = 0;
   for (size_t i = 0; i < full_hashes.size(); ++i) {
     const SBPrefix prefix = full_hashes[i].prefix;
-    if ((use_prefix_set && prefix_set_->Exists(prefix)) ||
-        (!use_prefix_set && browse_bloom_filter_->Exists(prefix))) {
+    if (prefix_set_->Exists(prefix)) {
       prefix_hits->push_back(prefix);
       if (prefix_miss_cache_.count(prefix) > 0)
         ++miss_count;
     }
   }
 
   // If all the prefixes are cached as 'misses', don't issue a GetHash.
   if (miss_count == prefix_hits->size())
     return false;
 
@@ skipping 438 matching lines @@
     if (download_whitelist_store_.get() &&
         !download_whitelist_store_->CheckValidity()) {
       DLOG(ERROR) << "Safe-browsing download whitelist database corrupt.";
     }
   }
 
   if (corruption_detected_)
     return;
 
   // Unroll the transaction if there was a protocol error or if the
-  // transaction was empty.  This will leave the bloom filter, the
+  // transaction was empty.  This will leave the prefix set, the
   // pending hashes, and the prefix miss cache in place.
   if (!update_succeeded || !change_detected_) {
     // Track empty updates to answer questions at http://crbug.com/72216 .
     if (update_succeeded && !change_detected_)
       UMA_HISTOGRAM_COUNTS("SB2.DatabaseUpdateKilobytes", 0);
     browse_store_->CancelUpdate();
     if (download_store_.get())
       download_store_->CancelUpdate();
     if (csd_whitelist_store_.get())
       csd_whitelist_store_->CancelUpdate();
@@ skipping 139 matching lines @@
     base::AutoLock locked(lookup_lock_);
     full_browse_hashes_.swap(add_full_hashes);
 
     // TODO(shess): If |CacheHashResults()| is posted between the
     // earlier lock and this clear, those pending hashes will be lost.
     // It could be fixed by only removing hashes which were collected
     // at the earlier point.  I believe that is fail-safe as-is (the
     // hash will be fetched again).
     pending_browse_hashes_.clear();
     prefix_miss_cache_.clear();
-    browse_bloom_filter_ = NULL;  // Stop using the bloom filter.
     prefix_set_.swap(prefix_set);
   }
 
   DVLOG(1) << "SafeBrowsingDatabaseImpl built prefix set in "
            << (base::TimeTicks::Now() - before).InMilliseconds()
            << " ms total.  prefix count: " << add_prefixes.size();
   UMA_HISTOGRAM_LONG_TIMES("SB2.BuildFilter", base::TimeTicks::Now() - before);
 
   // Persist the prefix set to disk.  Since only this thread changes
   // |prefix_set_|, there is no need to lock.
@@ skipping 40 matching lines @@
 
 void SafeBrowsingDatabaseNew::OnHandleCorruptDatabase() {
   RecordFailure(FAILURE_DATABASE_CORRUPT_HANDLER);
   corruption_detected_ = true;  // Stop updating the database.
   ResetDatabase();
   DLOG(FATAL) << "SafeBrowsing database was corrupt and reset";
 }
 
 // TODO(shess): I'm not clear why this code doesn't have any
 // real error-handling.
-// TODO(shess): After a transition period, this can convert to just
-// giving up if the prefix set is not on disk.
-void SafeBrowsingDatabaseNew::LoadBloomFilterOrPrefixSet() {
+void SafeBrowsingDatabaseNew::LoadPrefixSet() {
   DCHECK_EQ(creation_loop_, MessageLoop::current());
-  DCHECK(!bloom_filter_filename_.empty());
   DCHECK(!prefix_set_filename_.empty());
 
   // If there is no database, the filter cannot be used.
   base::PlatformFileInfo db_info;
   if (!file_util::GetFileInfo(browse_filename_, &db_info) || db_info.size == 0)
     return;
 
-  RecordFilterLoad(FILTER_LOAD);
-
-  // If there is no prefix set, or if the file is too old, check for a
-  // bloom filter.
-  // TODO(shess): The time check is in case this code gets reverted
-  // and re-landed.  It might be good to keep as a sanity check.
-  // Better would be to put the db's checksum in the filter file.
-  base::PlatformFileInfo prefix_set_info;
-  if (!file_util::GetFileInfo(prefix_set_filename_, &prefix_set_info) ||
-      prefix_set_info.size == 0 ||
-      prefix_set_info.last_modified < db_info.last_modified) {
-    // No prefix set.
-    prefix_set_.reset();
-
-    int64 file_size = GetFileSizeOrZero(bloom_filter_filename_);
-    if (!file_size) {
-      RecordFailure(FAILURE_DATABASE_FILTER_MISSING);
-      return;
-    }
-
-    const base::TimeTicks before = base::TimeTicks::Now();
-    browse_bloom_filter_ = BloomFilter::LoadFile(bloom_filter_filename_);
-    DVLOG(1) << "SafeBrowsingDatabaseNew read bloom filter in "
-             << (base::TimeTicks::Now() - before).InMilliseconds() << " ms";
-    UMA_HISTOGRAM_TIMES("SB2.BloomFilterLoad", base::TimeTicks::Now() - before);
-
-    if (!browse_bloom_filter_.get())
-      RecordFailure(FAILURE_DATABASE_FILTER_READ);
-    else
-      RecordFilterLoad(FILTER_LOADED_BLOOM_FILTER);
-
-    return;
-  }
-
-  // Once there is a prefix set stored, never use the bloom filter.
-  browse_bloom_filter_ = NULL;
-
-  // TODO(shess): The bloom filter file should have been deleted in
-  // WritePrefixSet(), unless this code is reverted and re-landed.
-  // Just paranoid.
-  file_util::Delete(bloom_filter_filename_, false);
+  // Cleanup any stale bloom filter (no longer used).
+  // TODO(shess): Track failure to delete?
+  FilePath bloom_filter_filename = BloomFilterForFilename(browse_filename_);
+  file_util::Delete(bloom_filter_filename, false);
 
   const base::TimeTicks before = base::TimeTicks::Now();
   prefix_set_.reset(safe_browsing::PrefixSet::LoadFile(prefix_set_filename_));
   DVLOG(1) << "SafeBrowsingDatabaseNew read prefix set in "
            << (base::TimeTicks::Now() - before).InMilliseconds() << " ms";
   UMA_HISTOGRAM_TIMES("SB2.PrefixSetLoad", base::TimeTicks::Now() - before);
 
   if (!prefix_set_.get())
     RecordFailure(FAILURE_DATABASE_PREFIX_SET_READ);
-  else
-    RecordFilterLoad(FILTER_LOADED_PREFIX_SET);
 }
 
 bool SafeBrowsingDatabaseNew::Delete() {
   DCHECK_EQ(creation_loop_, MessageLoop::current());
 
   const bool r1 = browse_store_->Delete();
   if (!r1)
     RecordFailure(FAILURE_DATABASE_STORE_DELETE);
 
   const bool r2 = download_store_.get() ? download_store_->Delete() : true;
   if (!r2)
     RecordFailure(FAILURE_DATABASE_STORE_DELETE);
 
   const bool r3 = csd_whitelist_store_.get() ?
       csd_whitelist_store_->Delete() : true;
   if (!r3)
     RecordFailure(FAILURE_DATABASE_STORE_DELETE);
 
   const bool r4 = download_whitelist_store_.get() ?
       download_whitelist_store_->Delete() : true;
   if (!r4)
     RecordFailure(FAILURE_DATABASE_STORE_DELETE);
 
-  const bool r5 = file_util::Delete(bloom_filter_filename_, false);
+  FilePath bloom_filter_filename = BloomFilterForFilename(browse_filename_);
+  const bool r5 = file_util::Delete(bloom_filter_filename, false);
   if (!r5)
     RecordFailure(FAILURE_DATABASE_FILTER_DELETE);
 
   const bool r6 = file_util::Delete(prefix_set_filename_, false);
   if (!r6)
     RecordFailure(FAILURE_DATABASE_PREFIX_SET_DELETE);
   return r1 && r2 && r3 && r4 && r5 && r6;
 }
 
 void SafeBrowsingDatabaseNew::WritePrefixSet() {
   DCHECK_EQ(creation_loop_, MessageLoop::current());
 
   if (!prefix_set_.get())
     return;
 
   const base::TimeTicks before = base::TimeTicks::Now();
   const bool write_ok = prefix_set_->WriteFile(prefix_set_filename_);
   DVLOG(1) << "SafeBrowsingDatabaseNew wrote prefix set in "
            << (base::TimeTicks::Now() - before).InMilliseconds() << " ms";
   UMA_HISTOGRAM_TIMES("SB2.PrefixSetWrite", base::TimeTicks::Now() - before);
 
   if (!write_ok)
     RecordFailure(FAILURE_DATABASE_PREFIX_SET_WRITE);
 
-  // Delete any stale bloom filter (checking before deleting is
-  // unlikely to be faster).
-  file_util::Delete(bloom_filter_filename_, false);
-
 #if defined(OS_MACOSX)
   base::mac::SetFileBackupExclusion(prefix_set_filename_);
 #endif
 }
 
 void SafeBrowsingDatabaseNew::WhitelistEverything(SBWhitelist* whitelist) {
   base::AutoLock locked(lookup_lock_);
   whitelist->second = true;
   whitelist->first.clear();
 }
@@ skipping 21 matching lines @@
   if (std::binary_search(new_whitelist.begin(), new_whitelist.end(),
                          kill_switch)) {
     // The kill switch is whitelisted hence we whitelist all URLs.
     WhitelistEverything(whitelist);
   } else {
     base::AutoLock locked(lookup_lock_);
     whitelist->second = false;
     whitelist->first.swap(new_whitelist);
   }
 }