Precache: Optionally rank resources-to-precache globally.

components/precache/core/precache_fetcher.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/precache/core/precache_fetcher.h"
 
 #include <algorithm>
 #include <limits>
+#include <set>
 #include <utility>
 #include <vector>
 
 #include "base/base64.h"
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/callback.h"
 #include "base/command_line.h"
 #include "base/compiler_specific.h"
 #include "base/containers/hash_tables.h"
@@ skipping 27 matching lines @@
 // The following flags are for privacy reasons. For example, if a user clears
 // their cookies, but a tracking beacon is prefetched and the beacon specifies
 // its source URL in a URL param, the beacon site would be able to rebuild a
 // profile of the user. All three flags should occur together, or not at all,
 // per
 // https://groups.google.com/a/chromium.org/d/topic/net-dev/vvcodRV6SdM/discussion.
 const int kNoTracking =
     net::LOAD_DO_NOT_SAVE_COOKIES | net::LOAD_DO_NOT_SEND_COOKIES |
     net::LOAD_DO_NOT_SEND_AUTH_DATA;
 
+// The maximum number of URLFetcher requests that can be on flight in parallel.

[bengr, 2016/10/18 18:57:20]

nit: on -> in

[twifkak, 2016/10/18 19:52:27]

Done.

+// Note that OnManifestFetchComplete and OnResourceFetchComplete perform
+// remove_if operations which are O(kMaxParallelFetches). Those should be
+// optimized before increasing this value significantly.
+const int kMaxParallelFetches = 10;
+
 namespace {
 
-// The maximum number of URLFetcher requests that can be on flight in parallel.
-const int kMaxParallelFetches = 10;
-
 // The maximum for the Precache.Fetch.ResponseBytes.* histograms. We set this to
 // a number we expect to be in the 99th percentile for the histogram, give or
 // take.
 const int kMaxResponseBytes = 500 * 1024 * 1024;
 
 GURL GetDefaultConfigURL() {
   const base::CommandLine& command_line =
       *base::CommandLine::ForCurrentProcess();
   if (command_line.HasSwitch(switches::kPrecacheConfigSettingsURL)) {
     return GURL(
@@ skipping 105 matching lines @@
     hashes.append(reinterpret_cast<const char*>(sha1_hash), kHashBytesSize);
   }
   base::Base64Encode(hashes, &hashes);
   return hashes;
 }
 
 // Retrieves the manifest info on the DB thread. Manifest info for each of the
 // hosts in |hosts_to_fetch|, is added to |hosts_info|.
 std::deque<ManifestHostInfo> RetrieveManifestInfo(
     const base::WeakPtr<PrecacheDatabase>& precache_database,
-    std::vector<std::string> hosts_to_fetch) {
+    std::vector<std::pair<std::string, int64_t>> hosts_to_fetch) {
+  VLOG(9) << "RetrieveManifestInfo";
   std::deque<ManifestHostInfo> hosts_info;
   if (!precache_database)
     return hosts_info;
 
   for (const auto& host : hosts_to_fetch) {
-    auto referrer_host_info = precache_database->GetReferrerHost(host);
+    auto referrer_host_info = precache_database->GetReferrerHost(host.first);
     if (referrer_host_info.id != PrecacheReferrerHostEntry::kInvalidId) {
       std::vector<GURL> used_urls, unused_urls;
       precache_database->GetURLListForReferrerHost(referrer_host_info.id,
                                                    &used_urls, &unused_urls);
       hosts_info.push_back(
-          ManifestHostInfo(referrer_host_info.manifest_id, host,
-                           GetResourceURLBase64Hash(used_urls),
+          ManifestHostInfo(referrer_host_info.manifest_id, host.first,
+                           host.second, GetResourceURLBase64Hash(used_urls),
                            GetResourceURLBase64Hash(unused_urls)));
     } else {
       hosts_info.push_back(
-          ManifestHostInfo(PrecacheReferrerHostEntry::kInvalidId, host,
-                           std::string(), std::string()));
+          ManifestHostInfo(PrecacheReferrerHostEntry::kInvalidId, host.first,
+                           host.second, std::string(), std::string()));
     }
   }
   return hosts_info;
 }
 
 PrecacheQuota RetrieveQuotaInfo(
     const base::WeakPtr<PrecacheDatabase>& precache_database) {
+  VLOG(9) << "RetrieveQuotaInfo";
   PrecacheQuota quota;
   if (precache_database) {
     quota = precache_database->GetQuota();
   }
   return quota;
 }
 
 // Returns true if the |quota| time has expired.
 bool IsQuotaTimeExpired(const PrecacheQuota& quota,
                         const base::Time& time_now) {
@@ skipping 134 matching lines @@
   // These may be unset in tests.
   if (!unfinished_work.has_start_time())
     return;
   base::TimeDelta time_to_fetch =
       base::Time::Now() -
       base::Time::FromInternalValue(unfinished_work.start_time());
   UMA_HISTOGRAM_CUSTOM_TIMES("Precache.Fetch.TimeToComplete", time_to_fetch,
                              base::TimeDelta::FromSeconds(1),
                              base::TimeDelta::FromHours(4), 50);
 
-  // Number of manifests for which we have downloaded all resources.
-  int manifests_completed =
-      unfinished_work.num_manifest_urls() - remaining_manifest_urls_to_fetch;
+  int num_total_resources = unfinished_work.num_resource_urls();
+  int percent_completed =
+      num_total_resources == 0

[bengr, 2016/10/18 18:57:20]

I wonder if we should be reporting num_total_resources == 0 separately.

[twifkak, 2016/10/18 19:52:27]

I'm amenable to that. My intuition is that this is small, but I don't have any
evidence. How about the overflow bucket? Currently unused.

[bengr, 2016/10/18 21:50:31]

sgtm

[twifkak, 2016/10/18 22:55:47]

Done.

+          ? 0
+          : (100 * (static_cast<double>(num_total_resources -
+                                        remaining_resource_urls_to_fetch) /
+                    num_total_resources));
 
-  // If there are resource URLs left to fetch, the last manifest is not yet
-  // completed.
-  if (remaining_resource_urls_to_fetch > 0)
-    --manifests_completed;
-
-  DCHECK_GE(manifests_completed, 0);
-  int percent_completed = unfinished_work.num_manifest_urls() == 0
-                              ? 0
-                              : (static_cast<double>(manifests_completed) /
-                                  unfinished_work.num_manifest_urls() * 100);
+  VLOG(6) << "Percent completed: " << percent_completed;
 
   UMA_HISTOGRAM_PERCENTAGE("Precache.Fetch.PercentCompleted",
                            percent_completed);
-    UMA_HISTOGRAM_CUSTOM_COUNTS("Precache.Fetch.ResponseBytes.Total",
-                                unfinished_work.total_bytes(),
-                                1, kMaxResponseBytes, 100);
+  UMA_HISTOGRAM_CUSTOM_COUNTS("Precache.Fetch.ResponseBytes.Total",
+                              unfinished_work.total_bytes(), 1,
+                              kMaxResponseBytes, 100);
   UMA_HISTOGRAM_CUSTOM_COUNTS("Precache.Fetch.ResponseBytes.Network",
-                              unfinished_work.network_bytes(),
-                              1, kMaxResponseBytes,
-                              100);
+                              unfinished_work.network_bytes(), 1,
+                              kMaxResponseBytes, 100);
 }
 
 // static
 std::string PrecacheFetcher::GetResourceURLBase64HashForTesting(
     const std::vector<GURL>& urls) {
   return GetResourceURLBase64Hash(urls);
 }
 
 PrecacheFetcher::PrecacheFetcher(
     net::URLRequestContextGetter* request_context,
     const GURL& config_url,
     const std::string& manifest_url_prefix,
+    bool global_ranking,
     std::unique_ptr<PrecacheUnfinishedWork> unfinished_work,
     uint32_t experiment_id,
     const base::WeakPtr<PrecacheDatabase>& precache_database,
     const scoped_refptr<base::SingleThreadTaskRunner>& db_task_runner,
     PrecacheFetcher::PrecacheDelegate* precache_delegate)
     : request_context_(request_context),
       config_url_(config_url),
       manifest_url_prefix_(manifest_url_prefix),
+      global_ranking_(global_ranking),
       precache_database_(precache_database),
       db_task_runner_(std::move(db_task_runner)),
       precache_delegate_(precache_delegate),
       pool_(kMaxParallelFetches),
       experiment_id_(experiment_id) {
   DCHECK(request_context_.get());  // Request context must be non-NULL.
   DCHECK(precache_delegate_);  // Precache delegate must be non-NULL.
 
   DCHECK_NE(GURL(), GetDefaultConfigURL())
       << "Could not determine the precache config settings URL.";
   DCHECK_NE(std::string(), GetDefaultManifestURLPrefix())
       << "Could not determine the default precache manifest URL prefix.";
   DCHECK(unfinished_work);
 
   // Copy resources to member variable as a convenience.
   // TODO(rajendrant): Consider accessing these directly from the proto, by
   // keeping track of the current resource index.
   for (const auto& resource : unfinished_work->resource()) {
     if (resource.has_url() && resource.has_top_host_name()) {
+      // Weight doesn't matter, as the resources have already been sorted by
+      // this point.
       resources_to_fetch_.emplace_back(GURL(resource.url()),
-                                       resource.top_host_name());
+                                       resource.top_host_name(), 0);
     }
   }
   unfinished_work_ = std::move(unfinished_work);
 }
 
 PrecacheFetcher::~PrecacheFetcher() {
 }
 
 std::unique_ptr<PrecacheUnfinishedWork> PrecacheFetcher::CancelPrecaching() {
   // This could get called multiple times, and it should be handled gracefully.
   if (!unfinished_work_)
     return nullptr;
 
   unfinished_work_->clear_resource();
   if (unfinished_work_->has_config_settings()) {
     // If config fetch is incomplete, |top_hosts_to_fetch_| will be empty and
     // top hosts should be left as is in |unfinished_work_|.
     unfinished_work_->clear_top_host();
-    for (const auto& top_host : top_hosts_to_fetch_) {
+    for (const auto& top_host : top_hosts_fetching_)
       unfinished_work_->add_top_host()->set_hostname(top_host.hostname);
-    }
+    for (const auto& top_host : top_hosts_to_fetch_)
+      unfinished_work_->add_top_host()->set_hostname(top_host.hostname);
+  }
+  for (const auto& resource : resources_fetching_) {
+    auto new_resource = unfinished_work_->add_resource();
+    new_resource->set_url(resource.url.spec());
+    new_resource->set_top_host_name(resource.referrer);
   }
   for (const auto& resource : resources_to_fetch_) {
     auto new_resource = unfinished_work_->add_resource();
-    new_resource->set_url(resource.first.spec());
-    new_resource->set_top_host_name(resource.second);
+    new_resource->set_url(resource.url.spec());
+    new_resource->set_top_host_name(resource.referrer);
   }
-  for (const auto& it : pool_.elements()) {
-    const Fetcher* fetcher = it.first;
-    GURL config_url =
-        config_url_.is_empty() ? GetDefaultConfigURL() : config_url_;
-    if (fetcher->is_resource_request()) {
-      auto resource = unfinished_work_->add_resource();
-      resource->set_url(fetcher->url().spec());
-      resource->set_top_host_name(fetcher->referrer());
-    } else if (fetcher->url() != config_url) {
-      unfinished_work_->add_top_host()->set_hostname(fetcher->referrer());
-    }
-  }
+  top_hosts_fetching_.clear();
   top_hosts_to_fetch_.clear();
+  resources_fetching_.clear();
   resources_to_fetch_.clear();
   pool_.DeleteAll();
   return std::move(unfinished_work_);
 }
 
 void PrecacheFetcher::Start() {
   if (unfinished_work_->has_config_settings()) {
     DCHECK(unfinished_work_->has_start_time());
     DetermineManifests();
     return;
@@ skipping 10 matching lines @@
   VLOG(3) << "Fetching " << config_url;
   pool_.Add(base::MakeUnique<Fetcher>(
       request_context_.get(), config_url, std::string(),
       base::Bind(&PrecacheFetcher::OnConfigFetchComplete, AsWeakPtr()),
       false /* is_resource_request */, std::numeric_limits<int32_t>::max()));
 }
 
 void PrecacheFetcher::StartNextResourceFetch() {
   DCHECK(unfinished_work_->has_config_settings());
   while (!resources_to_fetch_.empty() && pool_.IsAvailable()) {
-    const auto& resource = resources_to_fetch_.front();
+    ResourceInfo& resource = resources_to_fetch_.front();
     const size_t max_bytes = std::min(
         quota_.remaining(),
         std::min(unfinished_work_->config_settings().max_bytes_per_resource(),
                  unfinished_work_->config_settings().max_bytes_total() -
                      unfinished_work_->total_bytes()));
-    VLOG(3) << "Fetching " << resource.first << " " << resource.second;
+    VLOG(3) << "Fetching " << resource.url << " " << resource.referrer;
     pool_.Add(base::MakeUnique<Fetcher>(
-        request_context_.get(), resource.first, resource.second,
+        request_context_.get(), resource.url, resource.referrer,
         base::Bind(&PrecacheFetcher::OnResourceFetchComplete, AsWeakPtr()),
         true /* is_resource_request */, max_bytes));
 
+    resources_fetching_.push_back(std::move(resource));
     resources_to_fetch_.pop_front();
   }
 }
 
-void PrecacheFetcher::StartNextManifestFetch() {
-  if (top_hosts_to_fetch_.empty() || !pool_.IsAvailable())
-    return;
-
-  // We only fetch one manifest at a time to keep the size of
-  // resources_to_fetch_ as small as possible.
-  VLOG(3) << "Fetching " << top_hosts_to_fetch_.front().manifest_url;
-  pool_.Add(base::MakeUnique<Fetcher>(
-      request_context_.get(), top_hosts_to_fetch_.front().manifest_url,
-      top_hosts_to_fetch_.front().hostname,
-      base::Bind(&PrecacheFetcher::OnManifestFetchComplete, AsWeakPtr()),
-      false /* is_resource_request */, std::numeric_limits<int32_t>::max()));
-  top_hosts_to_fetch_.pop_front();
+void PrecacheFetcher::StartNextManifestFetches() {
+  // We fetch as many manifests at a time as possible, as we need all resource
+  // URLs in memory in order to rank them.
+  while (!top_hosts_to_fetch_.empty() && pool_.IsAvailable()) {
+    ManifestHostInfo& top_host = top_hosts_to_fetch_.front();
+    VLOG(3) << "Fetching " << top_host.manifest_url;

[bengr, 2016/10/18 18:57:20]

Please remove all VLOG statements.

[twifkak, 2016/10/18 19:52:27]

Done.

+    pool_.Add(base::MakeUnique<Fetcher>(
+        request_context_.get(), top_host.manifest_url, top_host.hostname,
+        base::Bind(&PrecacheFetcher::OnManifestFetchComplete, AsWeakPtr(),
+                   top_host.visits),
+        false /* is_resource_request */, std::numeric_limits<int32_t>::max()));
+    top_hosts_fetching_.push_back(std::move(top_host));
+    top_hosts_to_fetch_.pop_front();
+  }
 }
 
 void PrecacheFetcher::NotifyDone(
     size_t remaining_manifest_urls_to_fetch,
     size_t remaining_resource_urls_to_fetch) {
+  VLOG(9) << "NotifyDone";
   RecordCompletionStatistics(*unfinished_work_,
                              remaining_manifest_urls_to_fetch,
                              remaining_resource_urls_to_fetch);
   precache_delegate_->OnDone();
 }
 
 void PrecacheFetcher::StartNextFetch() {
+  VLOG(9) << "StartNextFetch";
   DCHECK(unfinished_work_->has_config_settings());
 
   // If over the precache total size cap or daily quota, then stop prefetching.
   if ((unfinished_work_->total_bytes() >
        unfinished_work_->config_settings().max_bytes_total()) ||
       quota_.remaining() == 0) {
-    size_t pending_manifests_in_pool = 0;
-    size_t pending_resources_in_pool = 0;
-    for (const auto& element_pair : pool_.elements()) {
-      const Fetcher* fetcher = element_pair.first;
-      if (fetcher->is_resource_request())
-        pending_resources_in_pool++;
-      else if (fetcher->url() != config_url_)
-        pending_manifests_in_pool++;
-    }
     pool_.DeleteAll();
-    NotifyDone(top_hosts_to_fetch_.size() + pending_manifests_in_pool,
-               resources_to_fetch_.size() + pending_resources_in_pool);
+    NotifyDone(top_hosts_to_fetch_.size() + top_hosts_fetching_.size(),
+               resources_to_fetch_.size() + resources_fetching_.size());
     return;
   }
 
   StartNextResourceFetch();
-  StartNextManifestFetch();
+  StartNextManifestFetches();
   if (top_hosts_to_fetch_.empty() && resources_to_fetch_.empty() &&
       pool_.IsEmpty()) {
     // There are no more URLs to fetch, so end the precache cycle.
     NotifyDone(0, 0);
     // OnDone may have deleted this PrecacheFetcher, so don't do anything after
     // it is called.
   }
 }
 
 void PrecacheFetcher::OnConfigFetchComplete(const Fetcher& source) {
+  VLOG(9) << "OnConfigFetchComplete";
   UpdateStats(source.response_bytes(), source.network_response_bytes());
   if (source.network_url_fetcher() == nullptr) {
     pool_.DeleteAll();  // Cancel any other ongoing request.
   } else {
     // Attempt to parse the config proto. On failure, continue on with the
     // default configuration.
     ParseProtoFromFetchResponse(
         *source.network_url_fetcher(),
         unfinished_work_->mutable_config_settings());
     pool_.Delete(source);
     DetermineManifests();
   }
 }
 
 void PrecacheFetcher::DetermineManifests() {
   DCHECK(unfinished_work_->has_config_settings());
 
-  std::vector<std::string> top_hosts_to_fetch;
-  std::unique_ptr<std::deque<ManifestHostInfo>> top_hosts_info(
-      new std::deque<ManifestHostInfo>);
+  std::vector<std::pair<std::string, int64_t>> top_hosts_to_fetch;
   // Keep track of manifest URLs that are being fetched, in order to elide
   // duplicates.
   std::set<base::StringPiece> seen_top_hosts;
   int64_t rank = 0;
 
   for (const auto& host : unfinished_work_->top_host()) {
     ++rank;
     if (rank > unfinished_work_->config_settings().top_sites_count())
       break;
     if (seen_top_hosts.insert(host.hostname()).second)
-      top_hosts_to_fetch.push_back(host.hostname());
+      top_hosts_to_fetch.emplace_back(host.hostname(), host.visits());
   }
 
   // Attempt to fetch manifests for starting hosts up to the maximum top sites
   // count. If a manifest does not exist for a particular starting host, then
   // the fetch will fail, and that starting host will be ignored. Starting
   // hosts are not added if this is a continuation from a previous precache
   // session.
   if (resources_to_fetch_.empty()) {
     for (const std::string& host :
          unfinished_work_->config_settings().forced_site()) {
+      // We add a forced site with visits == 0, which means its resources will
+      // be downloaded last. TODO(twifkak): Consider removing support for
+      // forced_site.
       if (seen_top_hosts.insert(host).second)
-        top_hosts_to_fetch.push_back(host);
+        top_hosts_to_fetch.emplace_back(host, 0);
     }
   }
-  // We only fetch one manifest at a time to keep the size of
-  // resources_to_fetch_ as small as possible.
+  // We retrieve manifest usage and quota info from the local database before
+  // fetching the manifests.
   PostTaskAndReplyWithResult(
       db_task_runner_.get(), FROM_HERE,
       base::Bind(&RetrieveManifestInfo, precache_database_,
                  std::move(top_hosts_to_fetch)),
       base::Bind(&PrecacheFetcher::OnManifestInfoRetrieved, AsWeakPtr()));
 }
 
 void PrecacheFetcher::OnManifestInfoRetrieved(
     std::deque<ManifestHostInfo> manifests_info) {
+  VLOG(9) << "OnManifestInfoRetrieved";
   const std::string prefix = manifest_url_prefix_.empty()
                                  ? GetDefaultManifestURLPrefix()
                                  : manifest_url_prefix_;
   if (!GURL(prefix).is_valid()) {
     // Don't attempt to fetch any manifests if the manifest URL prefix
     // is invalid.
     top_hosts_to_fetch_.clear();
     unfinished_work_->set_num_manifest_urls(manifests_info.size());
-    NotifyDone(manifests_info.size(), resources_to_fetch_.size());
+    NotifyDone(manifests_info.size(), resources_to_rank_.size());
     return;
   }
 
   top_hosts_to_fetch_ = std::move(manifests_info);
   for (auto& manifest : top_hosts_to_fetch_) {
     manifest.manifest_url =
         GURL(prefix +
              net::EscapeQueryParamValue(
                  net::EscapeQueryParamValue(manifest.hostname, false), false));
     if (manifest.manifest_id != PrecacheReferrerHostEntry::kInvalidId) {
       manifest.manifest_url = net::AppendOrReplaceQueryParameter(
           manifest.manifest_url, "manifest",
           std::to_string(manifest.manifest_id));
       manifest.manifest_url = net::AppendOrReplaceQueryParameter(
           manifest.manifest_url, "used_resources", manifest.used_url_hash);
       manifest.manifest_url = net::AppendOrReplaceQueryParameter(
           manifest.manifest_url, "unused_resources", manifest.unused_url_hash);
       DCHECK(manifest.manifest_url.is_valid());
     }
   }
   unfinished_work_->set_num_manifest_urls(top_hosts_to_fetch_.size());
 
   PostTaskAndReplyWithResult(
       db_task_runner_.get(), FROM_HERE,
       base::Bind(&RetrieveQuotaInfo, precache_database_),
       base::Bind(&PrecacheFetcher::OnQuotaInfoRetrieved, AsWeakPtr()));
 }
 
 void PrecacheFetcher::OnQuotaInfoRetrieved(const PrecacheQuota& quota) {
+  VLOG(9) << "OnQuotaInfoRetrieved";
   quota_ = quota;
   base::Time time_now = base::Time::Now();
   if (IsQuotaTimeExpired(quota_, time_now)) {
     // This is a new day. Update daily quota, that starts today and expires by
     // end of today.
     quota_.set_start_time(time_now.LocalMidnight().ToInternalValue());
     quota_.set_remaining(
         unfinished_work_->config_settings().daily_quota_total());
     db_task_runner_->PostTask(
         FROM_HERE,
         base::Bind(&PrecacheDatabase::SaveQuota, precache_database_, quota_));
   }
   StartNextFetch();
 }
 
 ManifestHostInfo::ManifestHostInfo(int64_t manifest_id,
                                    const std::string& hostname,
+                                   int64_t visits,
                                    const std::string& used_url_hash,
                                    const std::string& unused_url_hash)
     : manifest_id(manifest_id),
       hostname(hostname),
+      visits(visits),
       used_url_hash(used_url_hash),
       unused_url_hash(unused_url_hash) {}
 
 ManifestHostInfo::~ManifestHostInfo() {}
 
 ManifestHostInfo::ManifestHostInfo(ManifestHostInfo&&) = default;
 
 ManifestHostInfo& ManifestHostInfo::operator=(ManifestHostInfo&&) = default;
 
-void PrecacheFetcher::OnManifestFetchComplete(const Fetcher& source) {
+ResourceInfo::ResourceInfo(const GURL& url,
+                           const std::string& referrer,
+                           double weight)
+    : url(url), referrer(referrer), weight(weight) {}
+
+ResourceInfo::~ResourceInfo() {}
+
+ResourceInfo::ResourceInfo(ResourceInfo&&) = default;
+
+ResourceInfo& ResourceInfo::operator=(ResourceInfo&&) = default;
+
+void PrecacheFetcher::OnManifestFetchComplete(int64_t host_visits,
+                                              const Fetcher& source) {
+  VLOG(9) << "OnManifestFetchComplete " << source.referrer();
   DCHECK(unfinished_work_->has_config_settings());
   UpdateStats(source.response_bytes(), source.network_response_bytes());
   if (source.network_url_fetcher() == nullptr) {
     pool_.DeleteAll();  // Cancel any other ongoing request.
   } else {
     PrecacheManifest manifest;
 
     if (ParseProtoFromFetchResponse(*source.network_url_fetcher(), &manifest)) {
-      const int32_t len =
-          std::min(manifest.resource_size(),
-                   unfinished_work_->config_settings().top_resources_count());
+      int32_t len = manifest.resource_size();
+      if (!global_ranking_) {
+        len = std::min(
+            len, unfinished_work_->config_settings().top_resources_count());
+      }
       const uint64_t resource_bitset =
           GetResourceBitset(manifest, experiment_id_);
       for (int i = 0; i < len; ++i) {
         if (((0x1ULL << i) & resource_bitset) &&
             manifest.resource(i).has_url()) {
           GURL url(manifest.resource(i).url());
           if (url.is_valid()) {
-            resources_to_fetch_.emplace_back(url, source.referrer());
+            VLOG(9) << "Adding resource " << url.spec();
+            double weight = manifest.resource(i).weight_ratio() * host_visits;
+            if (weight >= unfinished_work_->config_settings().min_weight())
+              resources_to_rank_.emplace_back(url, source.referrer(), weight);
           }
         }
       }
       db_task_runner_->PostTask(
           FROM_HERE, base::Bind(&PrecacheDatabase::UpdatePrecacheReferrerHost,
                                 precache_database_, source.referrer(),
                                 manifest.id().id(), base::Time::Now()));
     }
   }
 
+  top_hosts_fetching_.remove_if([&source](const ManifestHostInfo& top_host) {
+    return top_host.manifest_url == source.url();
+  });
+
   pool_.Delete(source);
+
+  if (top_hosts_to_fetch_.empty() && top_hosts_fetching_.empty()) {
+    VLOG(9) << "Ranking resources.";
+    // Done fetching manifests. Now sort resources_to_rank_ into

[bengr, 2016/10/18 18:57:20]

Not a requirement, but this seems like it's getting a bit unwieldy. Could you
factor out the global ranking stuff? At the very least it will make it easier to
test.

[twifkak, 2016/10/18 19:52:27]

Okay, I extracted the inside of this if statement into a function, extracted a
ResourceWeight() function, and removed the `if (!global_ranking)` above, so
top_resources_count applies regardless. Should feel less spidery now.

+    // resources_to_fetch_, by descending weight. When StartNextFetch runs, it
+    // will begin fetching resources.
+    resources_to_fetch_ = std::move(resources_to_rank_);
+    if (global_ranking_) {
+      std::stable_sort(
+          resources_to_fetch_.begin(), resources_to_fetch_.end(),
+          [](const ResourceInfo& first, const ResourceInfo& second) {
+            return first.weight > second.weight;
+          });
+    }
+    // Truncate to size |total_resources_count|.
+    const size_t num_resources = std::min(
+        resources_to_fetch_.size(),
+        static_cast<size_t>(
+            unfinished_work_->config_settings().total_resources_count()));
+    resources_to_fetch_.erase(resources_to_fetch_.begin() + num_resources,
+                              resources_to_fetch_.end());
+    // Save denominator for PercentCompleted UMA.
+    unfinished_work_->set_num_resource_urls(resources_to_fetch_.size());
+  }
+
   StartNextFetch();
 }
 
 void PrecacheFetcher::OnResourceFetchComplete(const Fetcher& source) {
   UpdateStats(source.response_bytes(), source.network_response_bytes());
 
   db_task_runner_->PostTask(
       FROM_HERE,
       base::Bind(&PrecacheDatabase::RecordURLPrefetch, precache_database_,
                  source.url(), source.referrer(), base::Time::Now(),
                  source.was_cached(), source.response_bytes()));
 
+  resources_fetching_.remove_if([&source](const ResourceInfo& resource) {
+    return resource.url == source.url();
+  });
+
   pool_.Delete(source);
 
   // The resource has already been put in the cache during the fetch process, so
   // nothing more needs to be done for the resource.
   StartNextFetch();
 }
 
 void PrecacheFetcher::UpdateStats(int64_t response_bytes,
                                   int64_t network_response_bytes) {
   DCHECK_LE(0, response_bytes);
@@ skipping 12 matching lines @@
       remaining = 0;
     quota_.set_remaining(
         used_bytes > quota_.remaining() ? 0U : quota_.remaining() - used_bytes);
     db_task_runner_->PostTask(
         FROM_HERE,
         base::Bind(&PrecacheDatabase::SaveQuota, precache_database_, quota_));
   }
 }
 
 }  // namespace precache