Update visit analysis code:

chrome/browser/history/visit_database.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/browser/history/visit_database.h"
 
 #include <algorithm>
 #include <limits>
 #include <map>
 #include <set>
@@ skipping 57 matching lines @@
                    &VisitDatabase::VisitAnalysis::FetchHistory);
     }
   }
   void AddVisit(VisitRow* visit) {
     visits_to_process_.push_back(*visit);
     MaybeProcessVisits();
   }
 
  private:
   enum VA_EVENTS {
-    VA_VISIT = 0,
-    VA_PRERENDER_STARTED_1 = 1,
-    VA_PRERENDER_USED_1 = 2,
-    VA_PRERENDER_STARTED_3 = 3,
-    VA_PRERENDER_USED_3 = 4,
-    VA_PRERENDER_STARTED_5 = 5,
-    VA_PRERENDER_USED_5 = 6,
-    VA_EVENT_MAX = 7
+    VA_VISIT = 10,
+    VA_VISIT_EXCLUDE_BACK_FORWARD = 11,
+    VA_VISIT_EXCLUDE_HOME_PAGE = 12,
+    VA_VISIT_EXCLUDE_REDIRECT_CHAIN = 13,
+    VA_PRERENDER_STARTED_1 = 14,
+    VA_PRERENDER_USED_1 = 15,
+    VA_PRERENDER_STARTED_3 = 16,
+    VA_PRERENDER_USED_3 = 17,
+    VA_PRERENDER_STARTED_5 = 18,
+    VA_PRERENDER_USED_5 = 19,
+    VA_EVENT_MAX = 20
   };
+  bool IsBackForward(content::PageTransition transition) {

[dominich, 2012/03/02 22:09:01]

if this becomes production code, these should all be const methods.

+    return (transition & content::PAGE_TRANSITION_FORWARD_BACK) != 0;
+  }
+  bool IsHomePage(content::PageTransition transition) {
+    return (transition & content::PAGE_TRANSITION_HOME_PAGE) != 0;
+  }
+  bool IsIntermediateRedirect(content::PageTransition transition) {
+    return (transition & content::PAGE_TRANSITION_CHAIN_END) == 0;

[dominich, 2012/03/02 22:09:01]

do navigations that are not part of a redirect chain have this flag set? or
should you be checking for the existence of a redirect chain and that this is
the end transition?

+  }
+  bool ShouldExcludeTransitionForPrediction(
+      content::PageTransition transition) {
+    return IsBackForward(transition) || IsHomePage(transition) ||
+        IsIntermediateRedirect(transition);
+  }
   void RecordEvent(VA_EVENTS event, base::Time timestamp) {
     int64 ts = timestamp.ToInternalValue();
     if (time_event_bitmaps_.count(ts) < 1)
       time_event_bitmaps_[ts] = 0;
     int mask = (1 << event);
     VLOG(1) << "event " << ts << " " << event;
     if (time_event_bitmaps_[ts] & mask) {
       VLOG(1) << "duplicate event";
       return;
     }
@@ skipping 79 matching lines @@
       return false;
     }
     VisitAnalysis* visit_analysis_;
     int num_prerenders_;
     std::vector<PrerenderData> prerenders_;
     VA_EVENTS prerender_event_;
     VA_EVENTS prerender_used_event_;
   };
   void ProcessVisit(const VisitRow& visit) {
     VLOG(1) << "processing visit to " << visit.url_id;
+    if (IsBackForward(visit.transition)) {
+      RecordEvent(VA_VISIT_EXCLUDE_BACK_FORWARD, visit.visit_time);
+      return;
+    }
+    if (IsHomePage(visit.transition)) {
+      RecordEvent(VA_VISIT_EXCLUDE_HOME_PAGE, visit.visit_time);
+      return;
+    }
+    if (IsIntermediateRedirect(visit.transition)) {
+      RecordEvent(VA_VISIT_EXCLUDE_REDIRECT_CHAIN, visit.visit_time);
+      return;
+    }
     RecordEvent(VA_VISIT, visit.visit_time);
+    UMA_HISTOGRAM_ENUMERATION(
+        "Prerender.LocalVisitCoreTransition",
+        visit.transition & content::PAGE_TRANSITION_CORE_MASK,
+        content::PAGE_TRANSITION_LAST_CORE + 1);
     for (int i = 0; i < static_cast<int>(emulated_prerender_managers_.size());
          i++) {
       emulated_prerender_managers_[i]->MaybeRecordPrerenderUsed(visit);
     }
     base::TimeDelta max_age =
         base::TimeDelta::FromSeconds(kPrerenderExpirationSeconds);
     base::TimeDelta min_age =
         base::TimeDelta::FromMilliseconds(kMinSpacingMilliseconds);
     std::set<URLID> currently_found;
     std::map<URLID, int> found;
     int num_occurrences = 0;
     base::Time last_started;
     URLID best_prerender = 0;
     int best_count = 0;
     for (int i = 0; i < static_cast<int>(visits_.size()); i++) {
-      VLOG(1) << visits_[i].visit_time.ToInternalValue()/1000000 << " "
-              << visits_[i].url_id;
-      if (visits_[i].url_id == visit.url_id) {
-        VLOG(1) << "****";
-        last_started = visits_[i].visit_time;
-        num_occurrences++;
-        currently_found.clear();
-        continue;
-      }
-      if (last_started > visits_[i].visit_time - max_age &&
-          last_started < visits_[i].visit_time - min_age) {
-        currently_found.insert(visits_[i].url_id);
-        VLOG(1) << "OK";
+      if (!ShouldExcludeTransitionForPrediction(visits_[i].transition)) {
+        VLOG(1) << visits_[i].visit_time.ToInternalValue()/1000000 << " "
+                  << visits_[i].url_id;
+        if (visits_[i].url_id == visit.url_id) {
+          VLOG(1) << "****";
+          last_started = visits_[i].visit_time;
+          num_occurrences++;
+          currently_found.clear();
+          continue;
+        }
+        if (last_started > visits_[i].visit_time - max_age &&
+            last_started < visits_[i].visit_time - min_age) {
+          currently_found.insert(visits_[i].url_id);
+          VLOG(1) << "OK";
+        }
+      } else {
+        VLOG(1) << i << " (skipping)";
       }
       if (i == static_cast<int>(visits_.size()) - 1 ||
           visits_[i+1].url_id == visit.url_id) {
         VLOG(1) << "#########";
         // output last window
         for (std::set<URLID>::iterator it = currently_found.begin();
              it != currently_found.end();
              ++it) {
           if (found.count(*it) < 1)
             found[*it] = 0;
@@ skipping 31 matching lines @@
       VisitRow visit = visits_to_process_[i];
       ProcessVisit(visit);
       visits_.push_back(visit);
     }
     visits_to_process_.clear();
     VLOG(1) << "done processing new visit.  new visit size = "
             << visits_.size();
   }
   void FetchHistory() {
     VLOG(1) << "VDB: fetching history";
-    visit_database_->GetAllVisitsInRange(base::Time(), base::Time(), 0,
-                                         &visits_);
+    visit_database_->GetAllVisitsInRange(base::Time(), base::Time(),
+                                         kMaxVisitRecords, &visits_);
     VLOG(1) << "READ visits " << visits_.size();
     if (logging::GetVlogVerbosity() >= 1) {
       for (int i = 0; i < static_cast<int>(visits_.size()); i++) {
         VLOG(1) << " Read visit: " << visits_[i].url_id;
       }
     }
     UMA_HISTOGRAM_COUNTS("Prerender.LocalVisitDatabaseSize", visits_.size());
 
     VLOG(1) << "--- all visits read.";
     visits_fetched_ = true;
     MaybeProcessVisits();
   }
+  static const int kMaxVisitRecords = 100 * 1000;

[dominich, 2012/03/02 22:09:01]

is there a chance that this will still cause OOM failures in some cases?

   static const int kFetchDelayMs = 30 * 1000;
   static const int kPrerenderExpirationSeconds = 300;
-  static const int kMinSpacingMilliseconds = 1000;
+  static const int kMinSpacingMilliseconds = 500;
   VisitDatabase* visit_database_;
   base::OneShotTimer<VisitAnalysis> timer_;
   VisitVector visits_;
   VisitVector visits_to_process_;
   bool visits_fetched_;
   std::map<int64, int> time_event_bitmaps_;
   std::vector<EmulatedPrerenderManager *> emulated_prerender_managers_;
 };
 
 VisitDatabase::VisitDatabase() : visit_analysis_(new VisitAnalysis(this)) {
@@ skipping 482 matching lines @@
     // Get the source entries out of the query result.
     while (statement.Step()) {
       std::pair<VisitID, VisitSource> source_entry(statement.ColumnInt64(0),
           static_cast<VisitSource>(statement.ColumnInt(1)));
       sources->insert(source_entry);
     }
   }
 }
 
 }  // namespace history