DuplicateContentResourceHandler to monitor resources and track how many times th…

content/browser/renderer_host/duplicate_resource_handler.cc

Index: content/browser/renderer_host/duplicate_resource_handler.cc
===================================================================
--- content/browser/renderer_host/duplicate_resource_handler.cc	(revision 0)
+++ content/browser/renderer_host/duplicate_resource_handler.cc	(revision 0)
@@ -0,0 +1,132 @@
+// Copyright (c) 2012 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/browser/renderer_host/duplicate_resource_handler.h"
+
+#include <set>
+
+#include "base/logging.h"
+#include "base/memory/singleton.h"
+#include "base/metrics/histogram.h"
+#include "content/browser/renderer_host/resource_request_info_impl.h"
+#include "net/base/io_buffer.h"
+#include "net/url_request/url_request.h"
+#include "third_party/smhasher/src/PMurHash.h"
+
+
+namespace content {
+
+namespace{
+
+class GlobalDuplicateRecords {
+
+  public:
+    static GlobalDuplicateRecords* GetInstance(){
+      return Singleton<GlobalDuplicateRecords>::get();
+    }
+    
+    std::set<uint32>* content_matches() {
+      return &content_matches_;
+    }
+    
+    std::set<uint32>* content_and_url_matches(){
+      return &content_and_url_matches_;
+    }
+
+  private:
+    friend class Singleton<GlobalDuplicateRecords>;
+    friend struct DefaultSingletonTraits<GlobalDuplicateRecords>;
+
+    GlobalDuplicateRecords() {}
+    ~GlobalDuplicateRecords() {}
+    
+    std::set<uint32> content_matches_;
+    std::set<uint32> content_and_url_matches_;
+};
+
+}  // namespace
+
+DuplicateResourceHandler::DuplicateResourceHandler(
+    scoped_ptr<ResourceHandler> next_handler, 
+    ResourceType::Type resource_type)
+    : LayeredResourceHandler(next_handler.Pass()),
+      resource_type_(resource_type),
+      pmurhash_ph1_(0),
+      pmurhash_pcarry_(0),
+      bytes_read_(0) {
+}
+
+DuplicateResourceHandler::~DuplicateResourceHandler() {
+}
+
+bool DuplicateResourceHandler::OnWillRead(int request_id, net::IOBuffer** buf, 
+                                          int* buf_size, int min_size) {
+  DCHECK_EQ(-1, min_size);
+  
+  if (!next_handler_->OnWillRead(request_id, buf, buf_size, min_size))
+    return false;
+  read_buffer_ = *buf; 
+  return true;
+}
+
+bool DuplicateResourceHandler::OnReadCompleted(int request_id, int bytes_read,
+                                              bool* defer) {
+
+  PMurHash32_Process(&pmurhash_ph1_, &pmurhash_pcarry_, read_buffer_->data(), bytes_read);
+  bytes_read_ += bytes_read;
+
+  return next_handler_->OnReadCompleted(request_id, bytes_read, defer);
+}
+
+bool DuplicateResourceHandler::OnResponseCompleted(
+    int request_id,
+    const net::URLRequestStatus& status,
+    const std::string& security_info) {
+  
+  if (status.status() != net::URLRequestStatus::SUCCESS)
+    return next_handler_->OnResponseCompleted(request_id, status, security_info);
+
+  uint32 resource_hash = PMurHash32_Result(pmurhash_ph1_, pmurhash_pcarry_, bytes_read_);
+
+  // Hash url into the resource to see whether it is from the same or 
+  // different url.
+  uint32 hashed_with_url;
+  const std::string url_spec = request_->url().spec();

[frankwang, 2012/07/19 16:10:27]

The segfault happens here.

[gavinp, 2012/07/19 16:14:13]

It seems you never initialize request_.

+  int url_length = url_spec.size();
+  PMurHash32_Process(&pmurhash_ph1_, &pmurhash_pcarry_, url_spec.data(), url_length);
+  hashed_with_url = PMurHash32_Result(pmurhash_ph1_, pmurhash_pcarry_, url_length + bytes_read_);
+  
+  DVLOG(4) << "url: " << url_spec;
+  DVLOG(4) << "resource hash: " << resource_hash;
+  DVLOG(4) << "hash with url: " << hashed_with_url;
+
+  std::set<uint32>* content_hashes = 
+      GlobalDuplicateRecords::GetInstance()->content_matches();
+  std::set<uint32>* content_and_url_hashes = 
+      GlobalDuplicateRecords::GetInstance()->content_and_url_matches();
+
+  const bool did_match_contents = content_hashes->count(resource_hash);
+  const bool did_match_contents_and_url = 
+      content_and_url_hashes->count(hashed_with_url);
+  
+  UMA_HISTOGRAM_BOOLEAN("Duplicate.Hits", did_match_contents);
+  UMA_HISTOGRAM_BOOLEAN("Duplicate.HitsSameUrl", did_match_contents &&
+                         did_match_contents_and_url);
+  if (did_match_contents && !did_match_contents_and_url) {
+      UMA_HISTOGRAM_CUSTOM_COUNTS("Duplicate.Size.HashHitUrlMiss", bytes_read_, 
+                                   1, 0x7FFFFFFF, 50);
+      UMA_HISTOGRAM_ENUMERATION("Duplicate.ResourceType.HashHitUrlMiss", resource_type_, 
+                                 ResourceType::LAST_TYPE);
+      content_and_url_hashes->insert(hashed_with_url);
+  } else {
+    content_hashes->insert(resource_hash);
+    content_and_url_hashes->insert(hashed_with_url);
+  }
+  
+  bytes_read_ = 0;
+  read_buffer_ = NULL;
+  return next_handler_->OnResponseCompleted(request_id, status, security_info);
+}
+
+}  //namespace content