Omnibox SearchProvider Experiment Client Implementation.

chrome/browser/autocomplete/search_provider.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/browser/autocomplete/search_provider.h"
 
 #include <algorithm>
 #include <cmath>
 
 #include "base/callback.h"
@@ skipping 72 matching lines @@
 // static
 const int SearchProvider::kDefaultProviderURLFetcherID = 1;
 // static
 const int SearchProvider::kKeywordProviderURLFetcherID = 2;
 // static
 bool SearchProvider::query_suggest_immediately_ = false;
 
 SearchProvider::SearchProvider(ACProviderListener* listener, Profile* profile)
     : AutocompleteProvider(listener, profile, "Search"),
       providers_(profile),
+      has_verbatim_relevance_(false),
+      verbatim_relevance_(0),
       suggest_results_pending_(0),
       have_suggest_results_(false),
       instant_finalized_(false) {
   // We use GetSuggestNumberOfGroups() as the group ID to mean "not in field
   // trial."  Field trial groups run from 0 to GetSuggestNumberOfGroups() - 1
   // (inclusive).
   int suggest_field_trial_group_number =
       AutocompleteFieldTrial::GetSuggestNumberOfGroups();
   if (AutocompleteFieldTrial::InSuggestFieldTrial()) {
     suggest_field_trial_group_number =
@@ skipping 37 matching lines @@
   for (ACMatches::iterator i = matches_.begin(); i != matches_.end();) {
     if (((i->type == AutocompleteMatch::SEARCH_HISTORY) ||
          (i->type == AutocompleteMatch::SEARCH_SUGGEST)) &&
         (i->fill_into_edit == text)) {
       i = matches_.erase(i);
     } else {
       ++i;
     }
   }
 
+  // TODO(msw): Update this or ensure it's not removed in
+  //            ConvertResultsToAutocompleteMatches?
   // Add the new suggest result. We give it a rank higher than
   // SEARCH_WHAT_YOU_TYPED so that it gets autocompleted.
   int did_not_accept_default_suggestion = default_suggest_results_.empty() ?
         TemplateURLRef::NO_SUGGESTIONS_AVAILABLE :
         TemplateURLRef::NO_SUGGESTION_CHOSEN;
   MatchMap match_map;
   AddMatchToMap(text, adjusted_input_text,
                 CalculateRelevanceForWhatYouTyped() + 1,
                 AutocompleteMatch::SEARCH_SUGGEST,
                 did_not_accept_default_suggestion, false,
                 input_.prevent_inline_autocomplete(), &match_map);
   DCHECK_EQ(1u, match_map.size());
   matches_.push_back(match_map.begin()->second);
 
   listener_->OnProviderUpdate(true);
 }
 
 void SearchProvider::Start(const AutocompleteInput& input,
                            bool minimal_changes) {
-  matches_.clear();
-
   instant_finalized_ =
       (input.matches_requested() != AutocompleteInput::ALL_MATCHES);
 
   // Can't return search/suggest results for bogus input or without a profile.
   if (!profile_ || (input.type() == AutocompleteInput::INVALID)) {
     Stop();
     return;
   }
 
   keyword_input_text_.clear();
@@ skipping 46 matching lines @@
     return;
   }
 
   input_ = input;
 
   DoHistoryQuery(minimal_changes);
   StartOrStopSuggestQuery(minimal_changes);
   ConvertResultsToAutocompleteMatches();
 }
 
+SearchProvider::SuggestResult::SuggestResult(const string16& suggestion,
+                                             bool has_suggested_relevance,
+                                             int relevance)
+    : suggestion(suggestion),
+      has_suggested_relevance(has_suggested_relevance),
+      relevance(relevance) {
+}
+
+SearchProvider::NavigationResult::NavigationResult(
+    const GURL& url,
+    const string16& site_name,
+    bool has_suggested_relevance,
+    int relevance)
+    : url(url),
+      site_name(site_name),
+      has_suggested_relevance(has_suggested_relevance),
+      relevance(relevance) {
+}
+
 class SearchProvider::CompareScoredTerms {
  public:
   bool operator()(const ScoredTerm& a, const ScoredTerm& b) {
     // Sort in descending relevance order.
     return a.second > b.second;
   }
 };
 
 void SearchProvider::Run() {
   // Start a new request with the current input.
@@ skipping 10 matching lines @@
         providers_.keyword_provider()->suggestions_url_ref(),
         keyword_input_text_));
   }
   // We should only get here if we have a suggest url for the keyword or default
   // providers.
   DCHECK_GT(suggest_results_pending_, 0);
 }
 
 void SearchProvider::Stop() {
   StopSuggest();
+  ClearResults();
   done_ = true;
   default_provider_suggest_text_.clear();
 }
 
 void SearchProvider::OnURLFetchComplete(const content::URLFetcher* source) {
   DCHECK(!done_);
   suggest_results_pending_--;
   DCHECK_GE(suggest_results_pending_, 0);  // Should never go negative.
   const net::HttpResponseHeaders* const response_headers =
       source->GetResponseHeaders();
@@ skipping 72 matching lines @@
         keyword_input_text_, num_matches, &keyword_history_results_);
   }
 }
 
 void SearchProvider::StartOrStopSuggestQuery(bool minimal_changes) {
   // Don't send any queries to the server until some time has elapsed after
   // the last keypress, to avoid flooding the server with requests we are
   // likely to end up throwing away anyway.
   const int kQueryDelayMs = 200;
 
+  // TODO(msw): Synchronously remove irrelevant inline autocomplete matches,
+  //            update the remaining matches for the new input.
+  //            Mark these matches from_previous = true?
+  for (ACMatches::iterator i = matches_.begin(); i != matches_.end();) {
+    if (i->contents.find(input_.text()) == string16::npos &&
+        i->description.find(input_.text()) == string16::npos) {
+      i = matches_.erase(i);
+    } else {
+      if (i->inline_autocomplete_offset != string16::npos) {

[Peter Kasting, 2012/05/04 00:11:33]

I wouldn't try to adjust the offsets directly, I'd just reconstruct the
matches... see below

[msw, 2012/05/04 09:43:40]

Done; but I still may need to update the results synchronously...

+        if (StartsWith(i->fill_into_edit, input_.text(), false))
+          i->inline_autocomplete_offset = input_.text().length();
+        else
+          i->inline_autocomplete_offset = string16::npos;
+      }
+      // TODO(msw): Fixup navigational result highlighting/styling?
+      ++i;
+    }
+  }
+
   if (!IsQuerySuitableForSuggest()) {
     StopSuggest();
+    ClearResults();
     return;
   }
 
   // For the minimal_changes case, if we finished the previous query and still
   // have its results, or are allowed to keep running it, just do that, rather
   // than starting a new query.
   if (minimal_changes &&
       (have_suggest_results_ ||
        (!done_ &&
         input_.matches_requested() == AutocompleteInput::ALL_MATCHES)))
     return;
 
   // We can't keep running any previous query, so halt it.
   StopSuggest();
+  ClearResults();
 
   // We can't start a new query if we're only allowed synchronous results.
   if (input_.matches_requested() != AutocompleteInput::ALL_MATCHES)
     return;
 
   // We'll have at least one pending fetch. Set it to 1 now, but the value is
   // correctly set in Run. As Run isn't invoked immediately we need to set this
   // now, else we won't think we're waiting on results from the server when we
   // really are.
   suggest_results_pending_ = 1;
@@ skipping 57 matching lines @@
 
   return true;
 }
 
 void SearchProvider::StopSuggest() {
   suggest_results_pending_ = 0;
   timer_.Stop();
   // Stop any in-progress URL fetches.
   keyword_fetcher_.reset();
   default_fetcher_.reset();
+}
+
+void SearchProvider::ClearResults() {
   keyword_suggest_results_.clear();
   default_suggest_results_.clear();
   keyword_navigation_results_.clear();
   default_navigation_results_.clear();
   have_suggest_results_ = false;
 }
 
 content::URLFetcher* SearchProvider::CreateSuggestFetcher(
     int id,
     const TemplateURLRef& suggestions_url,
@@ skipping 33 matching lines @@
     if (root_list->Get(2, &description_val) &&
         description_val->IsType(Value::TYPE_LIST))
       description_list = static_cast<ListValue*>(description_val);
   }
 
   // We don't care about the query URL list (the fourth element in the
   // response) for now.
 
   // Parse optional data in the results from the Suggest server if any.
   ListValue* type_list = NULL;
+  ListValue* relevance_list = NULL;
+  has_verbatim_relevance_ = false;
+  verbatim_relevance_ = 0;
   // 5th argument: Optional key-value pairs.
   // TODO: We may iterate the 5th+ arguments of the root_list if any other
   // optional data are defined.
   if (root_list->GetSize() > 4) {
     Value* optional_val;
     if (root_list->Get(4, &optional_val) &&
         optional_val->IsType(Value::TYPE_DICTIONARY)) {
       DictionaryValue* dict_val = static_cast<DictionaryValue*>(optional_val);
 
       // Parse Google Suggest specific type extension.
       const std::string kGoogleSuggestType("google:suggesttype");
       if (dict_val->HasKey(kGoogleSuggestType))
         dict_val->GetList(kGoogleSuggestType, &type_list);
+
+      // Parse Google Suggest specific relevance extension.
+      const std::string kGoogleSuggestRelevance("google:suggestrelevance");
+      if (dict_val->HasKey(kGoogleSuggestRelevance))
+        dict_val->GetList(kGoogleSuggestRelevance, &relevance_list);
+
+      // Parse Google Suggest specific verbatim relevance extension.
+      const std::string kGoogleVerbatimRelevance("google:verbatimrelevance");
+      if (dict_val->GetInteger(kGoogleVerbatimRelevance, &verbatim_relevance_))
+        has_verbatim_relevance_ = true;
     }
   }
 
   ListValue* result_list = static_cast<ListValue*>(result_val);
   for (size_t i = 0; i < result_list->GetSize(); ++i) {
     Value* suggestion_val;
     string16 suggestion_str;
     if (!result_list->Get(i, &suggestion_val) ||
         !suggestion_val->GetAsString(&suggestion_str))
       return false;
 
     // Google search may return empty suggestions for weird input characters,
     // they make no sense at all and can cause problem in our code.
     // See http://crbug.com/56214
     if (!suggestion_str.length())
       continue;
 
+    bool has_suggested_relevance = false;
+    int relevance = 0;
+    if (relevance_list && relevance_list->GetInteger(i, &relevance))
+      has_suggested_relevance = true;
+
     Value* type_val;
     std::string type_str;
     if (type_list && type_list->Get(i, &type_val) &&
         type_val->GetAsString(&type_str) && (type_str == "NAVIGATION")) {
       Value* site_val;
       string16 site_name;
       NavigationResults& navigation_results =
           is_keyword ? keyword_navigation_results_ :
                        default_navigation_results_;
       if ((navigation_results.size() < kMaxMatches) &&
           description_list && description_list->Get(i, &site_val) &&
           site_val->IsType(Value::TYPE_STRING) &&
           site_val->GetAsString(&site_name)) {
         // We can't blindly trust the URL coming from the server to be valid.
         GURL result_url(URLFixerUpper::FixupURL(UTF16ToUTF8(suggestion_str),
                                                 std::string()));
         if (result_url.is_valid()) {
-          navigation_results.push_back(NavigationResult(result_url, site_name));
+          navigation_results.push_back(NavigationResult(result_url, site_name,
+                                                        has_suggested_relevance,
+                                                        relevance));
         }
       }
     } else {
       // TODO(kochi): Currently we treat a calculator result as a query, but it
       // is better to have better presentation for caluculator results.
-      if (suggest_results->size() < kMaxMatches)
-        suggest_results->push_back(suggestion_str);
+      if (suggest_results->size() < kMaxMatches) {
+        suggest_results->push_back(SuggestResult(suggestion_str,
+                                                 has_suggested_relevance,
+                                                 relevance));
+      }
     }
   }
 
   return true;
 }
 
 void SearchProvider::ConvertResultsToAutocompleteMatches() {
   // Convert all the results to matches and add them to a map, so we can keep
   // the most relevant match for each result.
   MatchMap map;
   const Time no_time;
   int did_not_accept_keyword_suggestion = keyword_suggest_results_.empty() ?
       TemplateURLRef::NO_SUGGESTIONS_AVAILABLE :
       TemplateURLRef::NO_SUGGESTION_CHOSEN;
   // Keyword what you typed results are handled by the KeywordProvider.
 
   int did_not_accept_default_suggestion = default_suggest_results_.empty() ?
         TemplateURLRef::NO_SUGGESTIONS_AVAILABLE :
         TemplateURLRef::NO_SUGGESTION_CHOSEN;
   if (providers_.valid_default_provider()) {
-    AddMatchToMap(input_.text(), input_.text(),
-                  CalculateRelevanceForWhatYouTyped(),
-                  AutocompleteMatch::SEARCH_WHAT_YOU_TYPED,
-                  did_not_accept_default_suggestion, false,
-                  input_.prevent_inline_autocomplete(), &map);
+    int verbatim_relevance = CalculateRelevanceForWhatYouTyped();
+    if (verbatim_relevance > 0) {
+      AddMatchToMap(input_.text(), input_.text(), verbatim_relevance,
+                    AutocompleteMatch::SEARCH_WHAT_YOU_TYPED,
+                    did_not_accept_default_suggestion, false,
+                    input_.prevent_inline_autocomplete(), &map);
+    }
+    // TODO(msw): Should verbatim_relevance affect and/or supress this result?
     if (!default_provider_suggest_text_.empty()) {
       AddMatchToMap(input_.text() + default_provider_suggest_text_,
                     input_.text(), CalculateRelevanceForWhatYouTyped() + 1,
                     AutocompleteMatch::SEARCH_SUGGEST,
                     did_not_accept_default_suggestion, false,
                     input_.prevent_inline_autocomplete(), &map);
     }
   }
 
   AddHistoryResultsToMap(keyword_history_results_, true,
                          did_not_accept_keyword_suggestion, &map);
   AddHistoryResultsToMap(default_history_results_, false,
                          did_not_accept_default_suggestion, &map);
 
   AddSuggestResultsToMap(keyword_suggest_results_, true,
                          did_not_accept_keyword_suggestion, &map);
   AddSuggestResultsToMap(default_suggest_results_, false,
                          did_not_accept_default_suggestion, &map);
 
   // Now add the most relevant matches from the map to |matches_|.
-  matches_.clear();
+  // TODO(msw): Clear the old matches now?? (only if there are new ones to add?)
+  if (!map.empty())
+    matches_.clear();
+  // TODO(msw): Merge matches from_previous = true?

[Peter Kasting, 2012/05/04 00:11:33]

So remember that if you merge previous matches, you need to re-construct them at
least insofar as the inline autocomplete offset goes -- this is maybe why you
have stale autocompletion issues today.  If you're currently storing the full
old AutocompleteMatches, I'd just store the Suggestions (or whatever the data
structures are called) and reconstruct the matches here

[msw, 2012/05/04 09:43:40]

Done.

+  //            (I think removing and re-adding any repeats should be okay...)
   for (MatchMap::const_iterator i(map.begin()); i != map.end(); ++i)
     matches_.push_back(i->second);
 
+  // TODO(msw): Need additional logic to preserve the navigational matches from
+  //            StartOrStopSuggestQuery cleanup through the ClearResults calls?
   AddNavigationResultsToMatches(keyword_navigation_results_, true);
   AddNavigationResultsToMatches(default_navigation_results_, false);
 
+  // TODO(msw): Additional logic to only +1 if we have a what-you-typed?

[Peter Kasting, 2012/05/04 00:11:33]

Yeah

[msw, 2012/05/04 09:43:40]

Done.

   const size_t max_total_matches = kMaxMatches + 1;  // 1 for "what you typed"
   std::partial_sort(matches_.begin(),
       matches_.begin() + std::min(max_total_matches, matches_.size()),
       matches_.end(), &AutocompleteMatch::MoreRelevant);
   if (matches_.size() > max_total_matches)
     matches_.erase(matches_.begin() + max_total_matches, matches_.end());
 
   UpdateStarredStateOfMatches();
 
   UpdateDone();
 }
 
 void SearchProvider::AddNavigationResultsToMatches(
     const NavigationResults& navigation_results,
     bool is_keyword) {
   if (!navigation_results.empty()) {
     // TODO(kochi): http://b/1170574  We add only one results for navigational
     // suggestions. If we can get more useful information about the score,
     // consider adding more results.
-    const size_t num_results = is_keyword ?
-        keyword_navigation_results_.size() : default_navigation_results_.size();
     matches_.push_back(NavigationToMatch(navigation_results.front(),
-        CalculateRelevanceForNavigation(num_results, 0, is_keyword),
+        CalculateRelevanceForNavigation(navigation_results, 0, is_keyword),
         is_keyword));
   }
 }
 
 void SearchProvider::AddHistoryResultsToMap(const HistoryResults& results,
                                             bool is_keyword,
                                             int did_not_accept_suggestion,
                                             MatchMap* map) {
   if (results.empty())
     return;
@@ skipping 89 matching lines @@
 
   return scored_terms;
 }
 
 void SearchProvider::AddSuggestResultsToMap(
     const SuggestResults& suggest_results,
     bool is_keyword,
     int did_not_accept_suggestion,
     MatchMap* map) {
   for (size_t i = 0; i < suggest_results.size(); ++i) {
-    AddMatchToMap(suggest_results[i],
+    AddMatchToMap(suggest_results[i].suggestion,
                   is_keyword ? keyword_input_text_ : input_.text(),
-                  CalculateRelevanceForSuggestion(suggest_results.size(), i,
+                  CalculateRelevanceForSuggestion(suggest_results, i,
                                                   is_keyword),
                   AutocompleteMatch::SEARCH_SUGGEST,
                   static_cast<int>(i), is_keyword,
                   input_.prevent_inline_autocomplete(), map);
   }
 }
 
 int SearchProvider::CalculateRelevanceForWhatYouTyped() const {
+  // TODO(msw): Effect just the exact SEARCH_WHAT_YOU_TYPED match (line 624) or
+  //            also inline-autocomplete from FinalizeInstantQuery and
+  //            default_provider_suggest_text_???

[Peter Kasting, 2012/05/04 00:11:33]

It's not at all clear to me how instant is supposed to interact with these
experiments.  You should probably get the suggest team to clarify, maybe for now
they'll just say to turn the experiment off for instant users?

[msw, 2012/05/04 09:43:40]

I have a TODO on the CR & in the CL, will e-mail/ask tomorrow.

+
+  // TODO(msw): Do not demote on just_deleted_text_/prevent_inline_autocomplete?

[Peter Kasting, 2012/05/04 00:11:33]

I think your code does this currently and is correct to do so

[msw, 2012/05/04 09:43:40]

Cool, thanks for the confirmation.

+  if (has_verbatim_relevance_ && !input_.prevent_inline_autocomplete())
+    return verbatim_relevance_;
+
   if (providers_.valid_keyword_provider())
     return 250;
 
   switch (input_.type()) {
     case AutocompleteInput::UNKNOWN:
     case AutocompleteInput::QUERY:
     case AutocompleteInput::FORCED_QUERY:
       return 1300;
 
     case AutocompleteInput::REQUESTED_URL:
@@ skipping 36 matching lines @@
   // Don't let scores go below 0.  Negative relevance scores are meaningful in
   // a different way.
   int base_score;
   if (is_primary_provider)
     base_score = (input_.type() == AutocompleteInput::URL) ? 750 : 1050;
   else
     base_score = 200;
   return std::max(0, base_score - score_discount);
 }
 
-int SearchProvider::CalculateRelevanceForSuggestion(size_t num_results,
-                                                    size_t result_number,
-                                                    bool is_keyword) const {
-  DCHECK(result_number < num_results);
+int SearchProvider::CalculateRelevanceForSuggestion(
+    const SuggestResults& results,
+    size_t result_number,
+    bool is_keyword) const {
+  DCHECK(result_number < results.size());
+  if (results[result_number].has_suggested_relevance)
+    return results[result_number].relevance;
+
   int base_score;
   if (!providers_.is_primary_provider(is_keyword))
     base_score = 100;
   else
     base_score = (input_.type() == AutocompleteInput::URL) ? 300 : 600;
   return base_score +
-      static_cast<int>(num_results - 1 - result_number);
+      static_cast<int>(results.size() - 1 - result_number);
 }
 
-int SearchProvider::CalculateRelevanceForNavigation(size_t num_results,
-                                                    size_t result_number,
-                                                    bool is_keyword) const {
-  DCHECK(result_number < num_results);
-  // TODO(kochi): http://b/784900  Use relevance score from the NavSuggest
-  // server if possible.
+int SearchProvider::CalculateRelevanceForNavigation(
+    const NavigationResults& results,
+    size_t result_number,
+    bool is_keyword) const {
+  DCHECK(result_number < results.size());
+  if (results[result_number].has_suggested_relevance)
+    return results[result_number].relevance;
+
   return (providers_.is_primary_provider(is_keyword) ? 800 : 150) +
-      static_cast<int>(num_results - 1 - result_number);
+      static_cast<int>(results.size() - 1 - result_number);
 }
 
 void SearchProvider::AddMatchToMap(const string16& query_string,
                                    const string16& input_text,
                                    int relevance,
                                    AutocompleteMatch::Type type,
                                    int accepted_suggestion,
                                    bool is_keyword,
                                    bool prevent_inline_autocomplete,
                                    MatchMap* map) {
@@ skipping 110 matching lines @@
                                            &match.description_class);
 
   // When the user forced a query, we need to make sure all the fill_into_edit
   // values preserve that property.  Otherwise, if the user starts editing a
   // suggestion, non-Search results will suddenly appear.
   if (input_.type() == AutocompleteInput::FORCED_QUERY)
     match.fill_into_edit.assign(ASCIIToUTF16("?"));
   match.fill_into_edit.append(
       AutocompleteInput::FormattedStringWithEquivalentMeaning(navigation.url,
                                                               match.contents));
-  // TODO(pkasting): http://b/1112879 These should perhaps be
+  // TODO(pkasting|msw): http://b/1112879 These should perhaps be
   // inline-autocompletable?
 
   return match;
 }
 
 void SearchProvider::UpdateDone() {
   // We're done when there are no more suggest queries pending (this is set to 1
   // when the timer is started) and we're not waiting on instant.
   done_ = ((suggest_results_pending_ == 0) &&
            (instant_finalized_ || !InstantController::IsEnabled(profile_)));
 }